From 1d994d707e3f8749ed6a7302cb590a4090327221 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Feb 2023 11:11:57 -0500 Subject: [PATCH 001/421] Backport of add integration tests for troubleshoot into release/1.15.x (#16261) * backport of commit 74dcceeeb2c2053d71e00e2aa2615f4572492ed4 * cherry pick from PR 16223 --------- Co-authored-by: Maliz --- .../consul-container/libs/service/connect.go | 58 +++++++++++----- .../consul-container/libs/service/examples.go | 15 ++++ .../consul-container/libs/service/gateway.go | 15 ++++ .../consul-container/libs/service/helpers.go | 2 - .../consul-container/libs/service/service.go | 7 +- .../libs/topology/service_topology.go | 51 ++++++++++++++ .../test/observability/access_logs_test.go | 41 +---------- .../troubleshoot_upstream_test.go | 68 +++++++++++++++++++ 8 files changed, 196 insertions(+), 61 deletions(-) create mode 100644 test/integration/consul-container/libs/topology/service_topology.go create mode 100644 test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 3fa9bcbde252..ac4907d4e583 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -20,17 +20,33 @@ import ( // ConnectContainer type ConnectContainer struct { - ctx context.Context - container testcontainers.Container - ip string - appPort int - adminPort int - mappedPublicPort int - serviceName string + ctx context.Context + container testcontainers.Container + ip string + appPort int + externalAdminPort int + internalAdminPort int + mappedPublicPort int + serviceName string } var _ Service = (*ConnectContainer)(nil) +func (g ConnectContainer) Exec(ctx context.Context, cmd []string) (string, error) { + exitCode, reader, err := g.container.Exec(ctx, cmd) + if err != nil { + return "", fmt.Errorf("exec with error %s", err) + } + if exitCode != 0 { + return "", fmt.Errorf("exec with exit code %d", exitCode) + } + buf, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("error reading from exec output: %w", err) + } + return string(buf), nil +} + func (g ConnectContainer) Export(partition, peer string, client *api.Client) error { return fmt.Errorf("ConnectContainer export unimplemented") } @@ -97,8 +113,13 @@ func (g ConnectContainer) Terminate() error { return cluster.TerminateContainer(g.ctx, g.container, true) } +func (g ConnectContainer) GetInternalAdminAddr() (string, int) { + return "localhost", g.internalAdminPort +} + +// GetAdminAddr returns the external admin port func (g ConnectContainer) GetAdminAddr() (string, int) { - return "localhost", g.adminPort + return "localhost", g.externalAdminPort } func (g ConnectContainer) GetStatus() (string, error) { @@ -133,7 +154,7 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID } dockerfileCtx.BuildArgs = buildargs - adminPort, err := node.ClaimAdminPort() + internalAdminPort, err := node.ClaimAdminPort() if err != nil { return nil, err } @@ -146,7 +167,7 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID Cmd: []string{ "consul", "connect", "envoy", "-sidecar-for", serviceID, - "-admin-bind", fmt.Sprintf("0.0.0.0:%d", adminPort), + "-admin-bind", fmt.Sprintf("0.0.0.0:%d", internalAdminPort), "--", "--log-level", envoyLogLevel, }, @@ -182,7 +203,7 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID var ( appPortStr = strconv.Itoa(serviceBindPort) - adminPortStr = strconv.Itoa(adminPort) + adminPortStr = strconv.Itoa(internalAdminPort) ) info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{appPortStr, adminPortStr}) @@ -191,18 +212,19 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID } out := &ConnectContainer{ - ctx: ctx, - container: info.Container, - ip: info.IP, - appPort: info.MappedPorts[appPortStr].Int(), - adminPort: info.MappedPorts[adminPortStr].Int(), - serviceName: sidecarServiceName, + ctx: ctx, + container: info.Container, + ip: info.IP, + appPort: info.MappedPorts[appPortStr].Int(), + externalAdminPort: info.MappedPorts[adminPortStr].Int(), + internalAdminPort: internalAdminPort, + serviceName: sidecarServiceName, } fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %d\n", serviceID, out.appPort, serviceBindPort) fmt.Printf("NewConnectService sidecar: name %s, mapped admin port %d, admin port %d\n", - sidecarServiceName, out.adminPort, adminPort) + sidecarServiceName, out.externalAdminPort, internalAdminPort) return out, nil } diff --git a/test/integration/consul-container/libs/service/examples.go b/test/integration/consul-container/libs/service/examples.go index e4c1fd186a37..9d95f6e9099f 100644 --- a/test/integration/consul-container/libs/service/examples.go +++ b/test/integration/consul-container/libs/service/examples.go @@ -29,6 +29,21 @@ type exampleContainer struct { var _ Service = (*exampleContainer)(nil) +func (g exampleContainer) Exec(ctx context.Context, cmd []string) (string, error) { + exitCode, reader, err := g.container.Exec(ctx, cmd) + if err != nil { + return "", fmt.Errorf("exec with error %s", err) + } + if exitCode != 0 { + return "", fmt.Errorf("exec with exit code %d", exitCode) + } + buf, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("error reading from exec output: %w", err) + } + return string(buf), nil +} + func (g exampleContainer) Export(partition, peerName string, client *api.Client) error { config := &api.ExportedServicesConfigEntry{ Name: partition, diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 5da2281338c6..5fb3a36184b4 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -30,6 +30,21 @@ type gatewayContainer struct { var _ Service = (*gatewayContainer)(nil) +func (g gatewayContainer) Exec(ctx context.Context, cmd []string) (string, error) { + exitCode, reader, err := g.container.Exec(ctx, cmd) + if err != nil { + return "", fmt.Errorf("exec with error %s", err) + } + if exitCode != 0 { + return "", fmt.Errorf("exec with exit code %d", exitCode) + } + buf, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("error reading from exec output: %w", err) + } + return string(buf), nil +} + func (g gatewayContainer) Export(partition, peer string, client *api.Client) error { return fmt.Errorf("gatewayContainer export unimplemented") } diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index 54a16249eec1..55ad94bb7f6b 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -3,9 +3,7 @@ package service import ( "context" "fmt" - "github.com/hashicorp/consul/api" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) diff --git a/test/integration/consul-container/libs/service/service.go b/test/integration/consul-container/libs/service/service.go index 75a35a74a6ff..57a3539a6412 100644 --- a/test/integration/consul-container/libs/service/service.go +++ b/test/integration/consul-container/libs/service/service.go @@ -1,13 +1,18 @@ package service -import "github.com/hashicorp/consul/api" +import ( + "context" + "github.com/hashicorp/consul/api" +) // Service represents a process that will be registered with the // Consul catalog, including Consul components such as sidecars and gateways type Service interface { + Exec(ctx context.Context, cmd []string) (string, error) // Export a service to the peering cluster Export(partition, peer string, client *api.Client) error GetAddr() (string, int) + // GetAdminAddr returns the external admin address GetAdminAddr() (string, int) GetLogs() (string, error) GetName() string diff --git a/test/integration/consul-container/libs/topology/service_topology.go b/test/integration/consul-container/libs/topology/service_topology.go new file mode 100644 index 000000000000..1cacd1a84c40 --- /dev/null +++ b/test/integration/consul-container/libs/topology/service_topology.go @@ -0,0 +1,51 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/stretchr/testify/require" +) + +func CreateServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service) { + node := cluster.Agents[0] + client := node.GetClient() + + // Register service as HTTP + serviceDefault := &api.ServiceConfigEntry{ + Kind: api.ServiceDefaults, + Name: libservice.StaticServerServiceName, + Protocol: "http", + } + + ok, _, err := client.ConfigEntries().Set(serviceDefault, nil) + require.NoError(t, err, "error writing HTTP service-default") + require.True(t, ok, "did not write HTTP service-default") + + // Create a service and proxy instance + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, + } + + // Create a service and proxy instance + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) + require.NoError(t, err) + + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName)) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + + // Create a client proxy instance with the server as an upstream + clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) + require.NoError(t, err) + + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + + return serverConnectProxy, clientConnectProxy +} diff --git a/test/integration/consul-container/test/observability/access_logs_test.go b/test/integration/consul-container/test/observability/access_logs_test.go index dcedb0de5531..6c173e7c0351 100644 --- a/test/integration/consul-container/test/observability/access_logs_test.go +++ b/test/integration/consul-container/test/observability/access_logs_test.go @@ -64,7 +64,7 @@ func TestAccessLogs(t *testing.T) { require.NoError(t, err) require.True(t, set) - serverService, clientService := createServices(t, cluster) + serverService, clientService := topology.CreateServices(t, cluster) _, port := clientService.GetAddr() // Validate Custom JSON @@ -121,42 +121,3 @@ func TestAccessLogs(t *testing.T) { // TODO: add a test to check that connections without a matching filter chain are NOT logged } - -func createServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service) { - node := cluster.Agents[0] - client := node.GetClient() - - // Register service as HTTP - serviceDefault := &api.ServiceConfigEntry{ - Kind: api.ServiceDefaults, - Name: libservice.StaticServerServiceName, - Protocol: "http", - } - - ok, _, err := client.ConfigEntries().Set(serviceDefault, nil) - require.NoError(t, err, "error writing HTTP service-default") - require.True(t, ok, "did not write HTTP service-default") - - // Create a service and proxy instance - serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", - HTTPPort: 8080, - GRPCPort: 8079, - } - - // Create a service and proxy instance - _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) - require.NoError(t, err) - - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName)) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) - - // Create a client proxy instance with the server as an upstream - clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) - require.NoError(t, err) - - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) - - return serverConnectProxy, clientConnectProxy -} diff --git a/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go new file mode 100644 index 000000000000..7803b38bff4f --- /dev/null +++ b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go @@ -0,0 +1,68 @@ +package troubleshoot + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" +) + +func TestTroubleshootProxy(t *testing.T) { + t.Parallel() + cluster, _, _ := topology.NewPeeringCluster(t, 1, &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + }) + + serverService, clientService := topology.CreateServices(t, cluster) + + clientSidecar, ok := clientService.(*libservice.ConnectContainer) + require.True(t, ok) + _, clientAdminPort := clientSidecar.GetInternalAdminAddr() + + t.Run("upstream exists and is healthy", func(t *testing.T) { + require.Eventually(t, func() bool { + output, err := clientSidecar.Exec(context.Background(), + []string{"consul", "troubleshoot", "upstreams", + "-envoy-admin-endpoint", fmt.Sprintf("localhost:%v", clientAdminPort)}) + require.NoError(t, err) + upstreamExists := assert.Contains(t, output, libservice.StaticServerServiceName) + + output, err = clientSidecar.Exec(context.Background(), []string{"consul", "troubleshoot", "proxy", + "-envoy-admin-endpoint", fmt.Sprintf("localhost:%v", clientAdminPort), + "-upstream-envoy-id", libservice.StaticServerServiceName}) + require.NoError(t, err) + certsValid := strings.Contains(output, "certificates are valid") + listenersExist := strings.Contains(output, fmt.Sprintf("listener for upstream \"%s\" found", libservice.StaticServerServiceName)) + routesExist := strings.Contains(output, fmt.Sprintf("route for upstream \"%s\" found", libservice.StaticServerServiceName)) + healthyEndpoints := strings.Contains(output, "✓ healthy endpoints for cluster") + return upstreamExists && certsValid && listenersExist && routesExist && healthyEndpoints + }, 60*time.Second, 10*time.Second) + }) + + t.Run("terminate upstream and check if client sees it as unhealthy", func(t *testing.T) { + err := serverService.Terminate() + require.NoError(t, err) + + require.Eventually(t, func() bool { + output, err := clientSidecar.Exec(context.Background(), []string{"consul", "troubleshoot", "proxy", + "-envoy-admin-endpoint", fmt.Sprintf("localhost:%v", clientAdminPort), + "-upstream-envoy-id", libservice.StaticServerServiceName}) + require.NoError(t, err) + + certsValid := strings.Contains(output, "certificates are valid") + listenersExist := strings.Contains(output, fmt.Sprintf("listener for upstream \"%s\" found", libservice.StaticServerServiceName)) + routesExist := strings.Contains(output, fmt.Sprintf("route for upstream \"%s\" found", libservice.StaticServerServiceName)) + endpointUnhealthy := strings.Contains(output, "no healthy endpoints for cluster") + return certsValid && listenersExist && routesExist && endpointUnhealthy + }, 60*time.Second, 10*time.Second) + }) +} From 6db3c26f54bc6f8ae3db2e947c27134e28573e23 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Feb 2023 11:30:44 -0500 Subject: [PATCH 002/421] backport of commit acf75497c876357fc2533e17a439adfc011e22fb (#16275) Co-authored-by: Nathan Coleman --- proto/pbconfigentry/config_entry.go | 16 + proto/pbconfigentry/config_entry.pb.go | 2225 ++++++++++++------------ proto/pbconfigentry/config_entry.proto | 1 + 3 files changed, 1141 insertions(+), 1101 deletions(-) diff --git a/proto/pbconfigentry/config_entry.go b/proto/pbconfigentry/config_entry.go index c570f9d35c6e..4b36134794d8 100644 --- a/proto/pbconfigentry/config_entry.go +++ b/proto/pbconfigentry/config_entry.go @@ -81,6 +81,14 @@ func ConfigEntryToStructs(s *ConfigEntry) structs.ConfigEntry { pbcommon.RaftIndexToStructs(s.RaftIndex, &target.RaftIndex) pbcommon.EnterpriseMetaToStructs(s.EnterpriseMeta, &target.EnterpriseMeta) return &target + case Kind_KindInlineCertificate: + var target structs.InlineCertificateConfigEntry + target.Name = s.Name + + InlineCertificateToStructs(s.GetInlineCertificate(), &target) + pbcommon.RaftIndexToStructs(s.RaftIndex, &target.RaftIndex) + pbcommon.EnterpriseMetaToStructs(s.EnterpriseMeta, &target.EnterpriseMeta) + return &target case Kind_KindServiceDefaults: var target structs.ServiceConfigEntry target.Name = s.Name @@ -177,6 +185,14 @@ func ConfigEntryFromStructs(s structs.ConfigEntry) *ConfigEntry { configEntry.Entry = &ConfigEntry_HTTPRoute{ HTTPRoute: &route, } + case *structs.InlineCertificateConfigEntry: + var cert InlineCertificate + InlineCertificateFromStructs(v, &cert) + + configEntry.Kind = Kind_KindInlineCertificate + configEntry.Entry = &ConfigEntry_InlineCertificate{ + InlineCertificate: &cert, + } default: panic(fmt.Sprintf("unable to convert %T to proto", s)) } diff --git a/proto/pbconfigentry/config_entry.pb.go b/proto/pbconfigentry/config_entry.pb.go index c929a21d5c51..4d9d292d7d44 100644 --- a/proto/pbconfigentry/config_entry.pb.go +++ b/proto/pbconfigentry/config_entry.pb.go @@ -575,6 +575,7 @@ type ConfigEntry struct { // *ConfigEntry_BoundAPIGateway // *ConfigEntry_TCPRoute // *ConfigEntry_HTTPRoute + // *ConfigEntry_InlineCertificate Entry isConfigEntry_Entry `protobuf_oneof:"Entry"` } @@ -708,6 +709,13 @@ func (x *ConfigEntry) GetHTTPRoute() *HTTPRoute { return nil } +func (x *ConfigEntry) GetInlineCertificate() *InlineCertificate { + if x, ok := x.GetEntry().(*ConfigEntry_InlineCertificate); ok { + return x.InlineCertificate + } + return nil +} + type isConfigEntry_Entry interface { isConfigEntry_Entry() } @@ -748,6 +756,10 @@ type ConfigEntry_HTTPRoute struct { HTTPRoute *HTTPRoute `protobuf:"bytes,13,opt,name=HTTPRoute,proto3,oneof"` } +type ConfigEntry_InlineCertificate struct { + InlineCertificate *InlineCertificate `protobuf:"bytes,14,opt,name=InlineCertificate,proto3,oneof"` +} + func (*ConfigEntry_MeshConfig) isConfigEntry_Entry() {} func (*ConfigEntry_ServiceResolver) isConfigEntry_Entry() {} @@ -766,6 +778,8 @@ func (*ConfigEntry_TCPRoute) isConfigEntry_Entry() {} func (*ConfigEntry_HTTPRoute) isConfigEntry_Entry() {} +func (*ConfigEntry_InlineCertificate) isConfigEntry_Entry() {} + // mog annotation: // // target=github.com/hashicorp/consul/agent/structs.MeshConfigEntry @@ -5310,7 +5324,7 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd2, 0x08, + 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbc, 0x09, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x3f, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, @@ -5379,815 +5393,686 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, - 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x22, 0xec, 0x03, 0x0a, 0x0a, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x6d, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x12, 0x46, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x49, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, - 0x65, 0x73, 0x68, 0x48, 0x54, 0x54, 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x04, 0x48, - 0x54, 0x54, 0x50, 0x12, 0x4f, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x68, 0x0a, 0x11, 0x49, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x0e, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, + 0x52, 0x11, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x22, 0xec, 0x03, 0x0a, + 0x0a, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x6d, 0x0a, 0x10, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x50, 0x0a, 0x1a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x32, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x4d, - 0x65, 0x73, 0x68, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4f, - 0x6e, 0x6c, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5b, 0x0a, 0x08, 0x49, 0x6e, 0x63, 0x6f, 0x6d, 0x69, 0x6e, - 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x65, + 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x46, 0x0a, 0x03, 0x54, 0x4c, + 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x4d, 0x65, 0x73, 0x68, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x54, - 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x49, 0x6e, 0x63, 0x6f, 0x6d, 0x69, - 0x6e, 0x67, 0x12, 0x5b, 0x0a, 0x08, 0x4f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, - 0x68, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x54, 0x4c, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x4f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x22, - 0x8a, 0x01, 0x0a, 0x18, 0x4d, 0x65, 0x73, 0x68, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, - 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, - 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, - 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x54, 0x0a, 0x0e, - 0x4d, 0x65, 0x73, 0x68, 0x48, 0x54, 0x54, 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, - 0x0a, 0x1c, 0x53, 0x61, 0x6e, 0x69, 0x74, 0x69, 0x7a, 0x65, 0x58, 0x46, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x65, 0x64, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x53, 0x61, 0x6e, 0x69, 0x74, 0x69, 0x7a, 0x65, 0x58, 0x46, - 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, - 0x72, 0x74, 0x22, 0x4d, 0x0a, 0x11, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, - 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x38, 0x0a, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, - 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, 0x68, - 0x72, 0x6f, 0x75, 0x67, 0x68, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x73, 0x22, 0xf6, 0x06, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x12, 0x24, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x44, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x5d, 0x0a, 0x07, 0x53, - 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, - 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x07, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x12, 0x5a, 0x0a, 0x08, 0x52, 0x65, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, - 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x52, 0x08, 0x52, 0x65, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x60, 0x0a, 0x08, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, - 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, - 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, - 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x41, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x57, 0x0a, 0x0c, 0x4c, - 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, - 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x52, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, - 0x6e, 0x63, 0x65, 0x72, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x78, 0x0a, 0x0c, 0x53, 0x75, - 0x62, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x52, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, + 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, + 0x4c, 0x53, 0x12, 0x49, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x48, 0x54, 0x54, + 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x04, 0x48, 0x54, 0x54, 0x50, 0x12, 0x4f, 0x0a, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, - 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7b, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x54, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, + 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, + 0x0a, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4d, + 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x50, 0x0a, 0x1a, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, + 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, 0x4d, 0x65, 0x73, + 0x68, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4f, 0x6e, 0x6c, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x44, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0xc9, 0x01, + 0x0a, 0x0d, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x5b, 0x0a, 0x08, 0x49, 0x6e, 0x63, 0x6f, 0x6d, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x44, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x08, 0x49, 0x6e, 0x63, 0x6f, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x5b, 0x0a, 0x08, + 0x4f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x44, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x08, 0x4f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x22, 0x8a, 0x01, 0x0a, 0x18, 0x4d, 0x65, + 0x73, 0x68, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x54, 0x4c, 0x53, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, + 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, + 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, + 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x54, 0x0a, 0x0e, 0x4d, 0x65, 0x73, 0x68, 0x48, 0x54, + 0x54, 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x1c, 0x53, 0x61, 0x6e, 0x69, + 0x74, 0x69, 0x7a, 0x65, 0x58, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, + 0x53, 0x61, 0x6e, 0x69, 0x74, 0x69, 0x7a, 0x65, 0x58, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, + 0x65, 0x64, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x22, 0x4d, 0x0a, 0x11, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x38, 0x0a, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, + 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x4d, + 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x73, 0x22, 0xf6, 0x06, 0x0a, 0x0f, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x12, + 0x24, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, + 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x5d, 0x0a, 0x07, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, - 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x15, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, - 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x4f, - 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xc9, 0x01, - 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, - 0x72, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, - 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, - 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, - 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, - 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, - 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, - 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x53, + 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x53, 0x75, 0x62, + 0x73, 0x65, 0x74, 0x73, 0x12, 0x5a, 0x0a, 0x08, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, - 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, - 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, - 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, - 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x12, 0x5d, 0x0a, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x69, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, - 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, - 0x69, 0x63, 0x79, 0x52, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, - 0x73, 0x22, 0x64, 0x0a, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, - 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, - 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, - 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, - 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, - 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, - 0xd3, 0x01, 0x0a, 0x0a, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, - 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, - 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, - 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x22, 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x2b, 0x0a, 0x03, 0x54, 0x54, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, - 0x50, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, - 0x22, 0x98, 0x03, 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, - 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x52, 0x08, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x12, 0x60, 0x0a, 0x08, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x08, 0x44, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, + 0x76, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, + 0x65, 0x72, 0x12, 0x41, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x57, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, + 0x52, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x54, + 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, + 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x78, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x52, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, + 0x73, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7b, + 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8f, 0x02, 0x0a, 0x14, - 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, - 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, - 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, - 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, - 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, - 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, - 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xea, 0x01, - 0x0a, 0x10, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x03, - 0x53, 0x44, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, - 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, - 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, - 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, - 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x51, 0x0a, 0x08, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x49, - 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xb7, 0x06, 0x0a, 0x0e, 0x49, 0x6e, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x79, 0x12, 0x54, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, - 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x73, 0x52, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x37, 0x0a, 0x09, 0x4d, + 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x15, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, + 0x73, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, + 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xc9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, + 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, + 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, + 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, + 0x65, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x12, + 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, + 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, + 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x12, 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, - 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, - 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, - 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, - 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, 0x17, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, - 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, + 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, + 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, + 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, + 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, + 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, + 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x5d, 0x0a, 0x0e, 0x52, 0x69, + 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x69, 0x6e, 0x67, 0x48, + 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, + 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, 0x0a, 0x12, 0x4c, 0x65, 0x61, + 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x65, + 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, + 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0c, 0x48, + 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0e, 0x52, + 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, + 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, + 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, + 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, + 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, + 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x43, 0x68, + 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x0a, 0x48, 0x61, + 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1e, + 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x57, + 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, 0x6f, + 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x22, + 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x03, 0x54, 0x54, 0x4c, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0x98, 0x03, 0x0a, 0x0e, 0x49, + 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x49, 0x0a, + 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, + 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, - 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, - 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x69, 0x65, 0x72, 0x73, 0x12, 0x55, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, - 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x53, + 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, + 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x37, 0x0a, 0x09, + 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8f, 0x02, 0x0a, 0x14, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x26, + 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, + 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x69, 0x65, 0x72, 0x73, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, - 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, - 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x50, 0x0a, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x12, 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0xa6, 0x06, 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, - 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, - 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, - 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, - 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, - 0x61, 0x63, 0x79, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, - 0x61, 0x63, 0x79, 0x49, 0x44, 0x12, 0x4e, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, + 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, - 0x79, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, + 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, + 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, + 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, + 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, + 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x51, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, + 0x54, 0x4c, 0x53, 0x22, 0xb7, 0x06, 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x48, 0x6f, + 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, + 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, + 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, + 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, + 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0e, 0x4d, + 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, + 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, + 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, + 0x17, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, + 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x55, + 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, - 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, - 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, - 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, - 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, - 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, - 0x0f, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, + 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x53, + 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, - 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x04, 0x48, 0x54, 0x54, 0x50, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, - 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, - 0x69, 0x78, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, - 0x12, 0x5c, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, - 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, - 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, - 0x0a, 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, - 0x0a, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, - 0x65, 0x67, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x22, 0xb6, 0x08, 0x0a, - 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, - 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x44, 0x0a, 0x04, - 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, + 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x07, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x56, 0x0a, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x69, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x5a, 0x0a, - 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, - 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4b, 0x0a, 0x06, 0x45, 0x78, 0x70, - 0x6f, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, - 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x53, 0x4e, 0x49, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x49, 0x12, 0x64, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, - 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5a, - 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x44, 0x65, 0x73, 0x74, - 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x44, - 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, - 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x3c, 0x0a, 0x19, - 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x19, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, - 0x74, 0x61, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, - 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, - 0x12, 0x5a, 0x0a, 0x0f, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x76, - 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x45, 0x6e, 0x76, - 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x09, - 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x74, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x32, 0x0a, 0x14, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x4f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, - 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, - 0x65, 0x63, 0x74, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x44, 0x69, 0x61, - 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6c, 0x79, 0x22, 0x5f, 0x0a, 0x11, 0x4d, - 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x4a, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x6f, 0x0a, 0x0c, - 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x0a, 0x06, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x73, 0x12, 0x47, 0x0a, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, - 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb0, 0x01, - 0x0a, 0x0a, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, - 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, - 0x68, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, 0x64, - 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x22, 0xbf, 0x01, 0x0a, 0x15, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x09, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, + 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, 0x06, + 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, + 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, + 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x12, + 0x4e, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, - 0x51, 0x0a, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x73, 0x22, 0x8a, 0x05, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x66, 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x18, + 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4c, 0x65, + 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x4c, + 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, + 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x12, 0x2c, 0x0a, 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, - 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, 0x4f, - 0x4e, 0x12, 0x2a, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x45, 0x6e, 0x76, - 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x1a, 0x0a, - 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x4d, 0x0a, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x06, 0x4c, 0x69, - 0x6d, 0x69, 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, - 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x5a, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, - 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, - 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x3e, 0x0a, 0x1a, 0x42, - 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x1a, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, - 0x65, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, - 0x9e, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, - 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, - 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, - 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x35, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x20, - 0x0a, 0x0b, 0x4d, 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x4d, 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, - 0x12, 0x38, 0x0a, 0x17, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, - 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x17, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x22, 0x45, 0x0a, 0x11, 0x44, 0x65, - 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x1c, 0x0a, 0x09, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x09, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, - 0x74, 0x22, 0xb6, 0x02, 0x0a, 0x0a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x12, 0x4f, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, + 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x4c, 0x65, 0x67, 0x61, 0x63, + 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4e, + 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, - 0x61, 0x12, 0x57, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, - 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, - 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, + 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, + 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x48, 0x54, + 0x54, 0x50, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, + 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x0a, + 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1c, 0x0a, 0x09, + 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x5c, 0x0a, 0x06, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x06, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x50, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x43, 0x6f, 0x6e, 0x64, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x54, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x68, + 0x6f, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, + 0x64, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x12, 0x16, 0x0a, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x65, 0x67, 0x65, + 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x16, + 0x0a, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, + 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x22, 0xb6, 0x08, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x44, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x12, 0x4c, 0x61, 0x73, 0x74, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x12, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x69, 0x6d, 0x65, 0x22, 0x8c, 0x02, 0x0a, 0x12, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, - 0x5d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x53, - 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, - 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, - 0x54, 0x4c, 0x53, 0x22, 0xde, 0x01, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, - 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, - 0x69, 0x74, 0x65, 0x73, 0x22, 0xb7, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4b, 0x69, - 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, - 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x69, 0x0a, 0x10, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x5a, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, - 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, - 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfe, - 0x01, 0x0a, 0x0f, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, - 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x5c, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x12, 0x4b, 0x0a, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, + 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x49, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, + 0x4e, 0x49, 0x12, 0x64, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5a, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, + 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, + 0x74, 0x4d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, + 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x3c, 0x0a, 0x19, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, + 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x42, 0x61, 0x6c, 0x61, 0x6e, + 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0d, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x5a, 0x0a, 0x0f, 0x45, 0x6e, + 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xdd, 0x01, 0x0a, 0x17, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x5c, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, - 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x50, 0x0a, - 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, + 0x74, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, 0x4f, 0x75, 0x74, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, + 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6c, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6c, 0x79, 0x22, 0x5f, 0x0a, 0x11, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x04, 0x4d, 0x6f, + 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, + 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x6f, 0x0a, 0x0c, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x12, 0x47, + 0x0a, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, - 0xe6, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x6c, 0x69, - 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x20, 0x0a, - 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, - 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, - 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x99, 0x03, 0x0a, 0x09, 0x48, 0x54, 0x54, - 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, - 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, + 0x52, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb0, 0x01, 0x0a, 0x0a, 0x45, 0x78, 0x70, 0x6f, + 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, + 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, + 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x50, 0x6f, 0x72, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x12, 0x28, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, + 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xbf, 0x01, 0x0a, 0x15, 0x55, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x09, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x05, 0x52, 0x75, - 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x52, - 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, + 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x08, 0x44, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x8a, 0x05, 0x0a, + 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x2c, 0x0a, + 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, + 0x4f, 0x4e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x2a, 0x0a, 0x10, 0x45, + 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x12, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, + 0x4d, 0x0a, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x69, + 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x5a, 0x0a, 0x0b, 0x4d, 0x65, 0x73, + 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x3e, 0x0a, 0x1a, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x42, 0x61, 0x6c, 0x61, 0x6e, + 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0x9e, 0x01, 0x0a, 0x0e, 0x55, 0x70, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, + 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x50, + 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x12, 0x35, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x4d, 0x61, 0x78, 0x46, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x4d, + 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x17, 0x45, 0x6e, + 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x76, 0x65, 0x35, 0x78, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x17, 0x45, 0x6e, 0x66, + 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, + 0x65, 0x35, 0x78, 0x78, 0x22, 0x45, 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xb6, 0x02, 0x0a, 0x0a, + 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4f, 0x0a, 0x04, 0x4d, 0x65, + 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x09, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x65, 0x72, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, @@ -6195,249 +6080,385 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf9, 0x01, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x73, 0x12, 0x4a, 0x0a, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, - 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, - 0x12, 0x4e, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x50, + 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x22, 0xc4, 0x02, 0x0a, 0x09, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, - 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x12, 0x4e, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, - 0x12, 0x48, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x8b, 0x02, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, + 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, + 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x54, 0x0a, 0x08, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x4b, 0x0a, 0x05, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x52, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, - 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x05, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x12, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x12, 0x4c, 0x61, 0x73, 0x74, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x8c, + 0x02, 0x0a, 0x12, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, + 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, + 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x5d, 0x0a, 0x08, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x41, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x53, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, + 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xde, 0x01, + 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0c, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x69, + 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, + 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, + 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0xb7, + 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, + 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, + 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, + 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, + 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfe, 0x01, 0x0a, 0x0f, 0x42, 0x6f, 0x75, + 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x54, 0x0a, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x75, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x50, - 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4e, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x72, 0x79, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, + 0x74, 0x61, 0x12, 0x5c, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x42, 0x6f, + 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, + 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, 0x17, 0x42, 0x6f, + 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x5c, 0x0a, 0x0c, 0x43, 0x65, 0x72, + 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x8b, - 0x01, 0x0a, 0x0e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x12, 0x4f, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb5, 0x01, 0x0a, - 0x0b, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x07, - 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0xe6, 0x01, 0x0a, 0x11, 0x49, 0x6e, + 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, + 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, - 0x53, 0x0a, 0x0b, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, - 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x52, 0x0b, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, - 0x69, 0x74, 0x65, 0x73, 0x22, 0x20, 0x0a, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, - 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0xc2, 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, - 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, - 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x12, 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, - 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, - 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, - 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, - 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, - 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, - 0xfc, 0x02, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x4d, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x69, + 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, + 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, + 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x99, 0x03, 0x0a, 0x09, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x12, 0x4e, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, + 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, + 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x45, + 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, - 0x01, 0x0a, 0x0a, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf9, + 0x01, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, + 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x2a, 0xfd, 0x01, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, - 0x4b, 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, - 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, - 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, - 0x6e, 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, 0x17, 0x0a, - 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, - 0x75, 0x6c, 0x74, 0x73, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, - 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, 0x6f, 0x75, - 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, 0x12, 0x11, - 0x0a, 0x0d, 0x4b, 0x69, 0x6e, 0x64, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, - 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x10, 0x0a, 0x2a, 0x26, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, - 0x12, 0x09, 0x0a, 0x05, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x2a, 0x50, - 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, - 0x00, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x10, 0x02, - 0x2a, 0x7b, 0x0a, 0x0f, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, - 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, - 0x17, 0x0a, 0x13, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, - 0x64, 0x65, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, - 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, 0x4f, 0x0a, - 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, 0x14, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x48, - 0x54, 0x54, 0x50, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, 0x2a, 0x92, - 0x02, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, - 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, - 0x65, 0x74, 0x68, 0x6f, 0x64, 0x41, 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, - 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, - 0x02, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, - 0x74, 0x68, 0x6f, 0x64, 0x47, 0x65, 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, - 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, 0x61, 0x64, - 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, - 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, 0x12, 0x18, - 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, - 0x64, 0x50, 0x61, 0x74, 0x63, 0x68, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, 0x74, 0x10, - 0x07, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, - 0x74, 0x68, 0x6f, 0x64, 0x50, 0x75, 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, - 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, 0x61, 0x63, - 0x65, 0x10, 0x09, 0x2a, 0xa7, 0x01, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x48, - 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, - 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, - 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, - 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, - 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, 0x68, 0x0a, - 0x11, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, - 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x10, 0x01, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, - 0x13, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, - 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, - 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, 0x68, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4a, + 0x0a, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x52, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x4e, 0x0a, 0x08, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, 0x43, - 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x22, 0xc4, 0x02, 0x0a, 0x09, 0x48, + 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4e, 0x0a, 0x06, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, + 0x6f, 0x64, 0x52, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x48, 0x0a, 0x04, 0x50, 0x61, + 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x04, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x4b, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, + 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0x75, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x12, 0x4e, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, + 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x0e, 0x48, 0x54, 0x54, + 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4f, 0x0a, 0x05, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, + 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb5, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, + 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x0b, 0x55, 0x52, 0x4c, + 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x52, 0x0b, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x22, 0x20, + 0x0a, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, + 0x22, 0xc2, 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x12, 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x03, 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, + 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, + 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, + 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, + 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfc, 0x02, 0x0a, 0x08, 0x54, 0x43, + 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, + 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, + 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x0a, 0x54, 0x43, 0x50, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x2a, 0xfd, 0x01, + 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x4b, 0x69, 0x6e, 0x64, 0x55, 0x6e, + 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x4d, + 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4b, + 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, + 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x10, 0x05, + 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x4b, + 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x07, 0x12, + 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, 0x12, 0x11, 0x0a, 0x0d, 0x4b, 0x69, 0x6e, 0x64, + 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x4b, + 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x0a, 0x2a, 0x26, 0x0a, + 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x6c, + 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, + 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x2a, 0x50, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, + 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x50, + 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, + 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x10, 0x02, 0x2a, 0x7b, 0x0a, 0x0f, 0x4d, 0x65, + 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, + 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, + 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4d, 0x65, 0x73, + 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4e, 0x6f, 0x6e, 0x65, + 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, + 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, 0x4f, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x48, 0x54, 0x54, 0x50, 0x10, 0x00, 0x12, + 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, 0x2a, 0x92, 0x02, 0x0a, 0x0f, 0x48, 0x54, 0x54, + 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x12, + 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x41, + 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x10, 0x01, + 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x48, + 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x47, 0x65, + 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, 0x61, 0x64, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, + 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x61, 0x74, 0x63, 0x68, + 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, 0x74, 0x10, 0x07, 0x12, 0x16, 0x0a, 0x12, 0x48, + 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x75, + 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, 0x61, 0x63, 0x65, 0x10, 0x09, 0x2a, 0xa7, 0x01, + 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, + 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, + 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, + 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, + 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x53, + 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, 0x68, 0x0a, 0x11, 0x48, 0x54, 0x54, 0x50, 0x50, + 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, + 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, + 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, 0x22, 0x0a, + 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, + 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, + 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, + 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x48, + 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, + 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, + 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, 0x43, 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -6563,118 +6584,119 @@ var file_proto_pbconfigentry_config_entry_proto_depIdxs = []int32{ 56, // 9: hashicorp.consul.internal.configentry.ConfigEntry.BoundAPIGateway:type_name -> hashicorp.consul.internal.configentry.BoundAPIGateway 69, // 10: hashicorp.consul.internal.configentry.ConfigEntry.TCPRoute:type_name -> hashicorp.consul.internal.configentry.TCPRoute 59, // 11: hashicorp.consul.internal.configentry.ConfigEntry.HTTPRoute:type_name -> hashicorp.consul.internal.configentry.HTTPRoute - 12, // 12: hashicorp.consul.internal.configentry.MeshConfig.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyMeshConfig - 13, // 13: hashicorp.consul.internal.configentry.MeshConfig.TLS:type_name -> hashicorp.consul.internal.configentry.MeshTLSConfig - 15, // 14: hashicorp.consul.internal.configentry.MeshConfig.HTTP:type_name -> hashicorp.consul.internal.configentry.MeshHTTPConfig - 71, // 15: hashicorp.consul.internal.configentry.MeshConfig.Meta:type_name -> hashicorp.consul.internal.configentry.MeshConfig.MetaEntry - 16, // 16: hashicorp.consul.internal.configentry.MeshConfig.Peering:type_name -> hashicorp.consul.internal.configentry.PeeringMeshConfig - 14, // 17: hashicorp.consul.internal.configentry.MeshTLSConfig.Incoming:type_name -> hashicorp.consul.internal.configentry.MeshDirectionalTLSConfig - 14, // 18: hashicorp.consul.internal.configentry.MeshTLSConfig.Outgoing:type_name -> hashicorp.consul.internal.configentry.MeshDirectionalTLSConfig - 72, // 19: hashicorp.consul.internal.configentry.ServiceResolver.Subsets:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry - 19, // 20: hashicorp.consul.internal.configentry.ServiceResolver.Redirect:type_name -> hashicorp.consul.internal.configentry.ServiceResolverRedirect - 73, // 21: hashicorp.consul.internal.configentry.ServiceResolver.Failover:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry - 91, // 22: hashicorp.consul.internal.configentry.ServiceResolver.ConnectTimeout:type_name -> google.protobuf.Duration - 22, // 23: hashicorp.consul.internal.configentry.ServiceResolver.LoadBalancer:type_name -> hashicorp.consul.internal.configentry.LoadBalancer - 74, // 24: hashicorp.consul.internal.configentry.ServiceResolver.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.MetaEntry - 21, // 25: hashicorp.consul.internal.configentry.ServiceResolverFailover.Targets:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailoverTarget - 23, // 26: hashicorp.consul.internal.configentry.LoadBalancer.RingHashConfig:type_name -> hashicorp.consul.internal.configentry.RingHashConfig - 24, // 27: hashicorp.consul.internal.configentry.LoadBalancer.LeastRequestConfig:type_name -> hashicorp.consul.internal.configentry.LeastRequestConfig - 25, // 28: hashicorp.consul.internal.configentry.LoadBalancer.HashPolicies:type_name -> hashicorp.consul.internal.configentry.HashPolicy - 26, // 29: hashicorp.consul.internal.configentry.HashPolicy.CookieConfig:type_name -> hashicorp.consul.internal.configentry.CookieConfig - 91, // 30: hashicorp.consul.internal.configentry.CookieConfig.TTL:type_name -> google.protobuf.Duration - 29, // 31: hashicorp.consul.internal.configentry.IngressGateway.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig - 31, // 32: hashicorp.consul.internal.configentry.IngressGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.IngressListener - 75, // 33: hashicorp.consul.internal.configentry.IngressGateway.Meta:type_name -> hashicorp.consul.internal.configentry.IngressGateway.MetaEntry - 28, // 34: hashicorp.consul.internal.configentry.IngressGateway.Defaults:type_name -> hashicorp.consul.internal.configentry.IngressServiceConfig - 48, // 35: hashicorp.consul.internal.configentry.IngressServiceConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 30, // 36: hashicorp.consul.internal.configentry.GatewayTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig - 32, // 37: hashicorp.consul.internal.configentry.IngressListener.Services:type_name -> hashicorp.consul.internal.configentry.IngressService - 29, // 38: hashicorp.consul.internal.configentry.IngressListener.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig - 33, // 39: hashicorp.consul.internal.configentry.IngressService.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayServiceTLSConfig - 34, // 40: hashicorp.consul.internal.configentry.IngressService.RequestHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers - 34, // 41: hashicorp.consul.internal.configentry.IngressService.ResponseHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers - 76, // 42: hashicorp.consul.internal.configentry.IngressService.Meta:type_name -> hashicorp.consul.internal.configentry.IngressService.MetaEntry - 89, // 43: hashicorp.consul.internal.configentry.IngressService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 48, // 44: hashicorp.consul.internal.configentry.IngressService.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 30, // 45: hashicorp.consul.internal.configentry.GatewayServiceTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig - 77, // 46: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.AddEntry - 78, // 47: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.SetEntry - 36, // 48: hashicorp.consul.internal.configentry.ServiceIntentions.Sources:type_name -> hashicorp.consul.internal.configentry.SourceIntention - 79, // 49: hashicorp.consul.internal.configentry.ServiceIntentions.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceIntentions.MetaEntry - 1, // 50: hashicorp.consul.internal.configentry.SourceIntention.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction - 37, // 51: hashicorp.consul.internal.configentry.SourceIntention.Permissions:type_name -> hashicorp.consul.internal.configentry.IntentionPermission - 2, // 52: hashicorp.consul.internal.configentry.SourceIntention.Type:type_name -> hashicorp.consul.internal.configentry.IntentionSourceType - 80, // 53: hashicorp.consul.internal.configentry.SourceIntention.LegacyMeta:type_name -> hashicorp.consul.internal.configentry.SourceIntention.LegacyMetaEntry - 92, // 54: hashicorp.consul.internal.configentry.SourceIntention.LegacyCreateTime:type_name -> google.protobuf.Timestamp - 92, // 55: hashicorp.consul.internal.configentry.SourceIntention.LegacyUpdateTime:type_name -> google.protobuf.Timestamp - 89, // 56: hashicorp.consul.internal.configentry.SourceIntention.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 1, // 57: hashicorp.consul.internal.configentry.IntentionPermission.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction - 38, // 58: hashicorp.consul.internal.configentry.IntentionPermission.HTTP:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPPermission - 39, // 59: hashicorp.consul.internal.configentry.IntentionHTTPPermission.Header:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPHeaderPermission - 3, // 60: hashicorp.consul.internal.configentry.ServiceDefaults.Mode:type_name -> hashicorp.consul.internal.configentry.ProxyMode - 41, // 61: hashicorp.consul.internal.configentry.ServiceDefaults.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyConfig - 42, // 62: hashicorp.consul.internal.configentry.ServiceDefaults.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig - 43, // 63: hashicorp.consul.internal.configentry.ServiceDefaults.Expose:type_name -> hashicorp.consul.internal.configentry.ExposeConfig - 45, // 64: hashicorp.consul.internal.configentry.ServiceDefaults.UpstreamConfig:type_name -> hashicorp.consul.internal.configentry.UpstreamConfiguration - 49, // 65: hashicorp.consul.internal.configentry.ServiceDefaults.Destination:type_name -> hashicorp.consul.internal.configentry.DestinationConfig - 81, // 66: hashicorp.consul.internal.configentry.ServiceDefaults.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceDefaults.MetaEntry - 93, // 67: hashicorp.consul.internal.configentry.ServiceDefaults.EnvoyExtensions:type_name -> hashicorp.consul.internal.common.EnvoyExtension - 4, // 68: hashicorp.consul.internal.configentry.MeshGatewayConfig.Mode:type_name -> hashicorp.consul.internal.configentry.MeshGatewayMode - 44, // 69: hashicorp.consul.internal.configentry.ExposeConfig.Paths:type_name -> hashicorp.consul.internal.configentry.ExposePath - 46, // 70: hashicorp.consul.internal.configentry.UpstreamConfiguration.Overrides:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig - 46, // 71: hashicorp.consul.internal.configentry.UpstreamConfiguration.Defaults:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig - 89, // 72: hashicorp.consul.internal.configentry.UpstreamConfig.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 47, // 73: hashicorp.consul.internal.configentry.UpstreamConfig.Limits:type_name -> hashicorp.consul.internal.configentry.UpstreamLimits - 48, // 74: hashicorp.consul.internal.configentry.UpstreamConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 42, // 75: hashicorp.consul.internal.configentry.UpstreamConfig.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig - 91, // 76: hashicorp.consul.internal.configentry.PassiveHealthCheck.Interval:type_name -> google.protobuf.Duration - 82, // 77: hashicorp.consul.internal.configentry.APIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.APIGateway.MetaEntry - 53, // 78: hashicorp.consul.internal.configentry.APIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.APIGatewayListener - 51, // 79: hashicorp.consul.internal.configentry.APIGateway.Status:type_name -> hashicorp.consul.internal.configentry.Status - 52, // 80: hashicorp.consul.internal.configentry.Status.Conditions:type_name -> hashicorp.consul.internal.configentry.Condition - 55, // 81: hashicorp.consul.internal.configentry.Condition.Resource:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 92, // 82: hashicorp.consul.internal.configentry.Condition.LastTransitionTime:type_name -> google.protobuf.Timestamp - 5, // 83: hashicorp.consul.internal.configentry.APIGatewayListener.Protocol:type_name -> hashicorp.consul.internal.configentry.APIGatewayListenerProtocol - 54, // 84: hashicorp.consul.internal.configentry.APIGatewayListener.TLS:type_name -> hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration - 55, // 85: hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 89, // 86: hashicorp.consul.internal.configentry.ResourceReference.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 83, // 87: hashicorp.consul.internal.configentry.BoundAPIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.BoundAPIGateway.MetaEntry - 57, // 88: hashicorp.consul.internal.configentry.BoundAPIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.BoundAPIGatewayListener - 55, // 89: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 55, // 90: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Routes:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 84, // 91: hashicorp.consul.internal.configentry.InlineCertificate.Meta:type_name -> hashicorp.consul.internal.configentry.InlineCertificate.MetaEntry - 85, // 92: hashicorp.consul.internal.configentry.HTTPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.HTTPRoute.MetaEntry - 55, // 93: hashicorp.consul.internal.configentry.HTTPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 60, // 94: hashicorp.consul.internal.configentry.HTTPRoute.Rules:type_name -> hashicorp.consul.internal.configentry.HTTPRouteRule - 51, // 95: hashicorp.consul.internal.configentry.HTTPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status - 65, // 96: hashicorp.consul.internal.configentry.HTTPRouteRule.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters - 61, // 97: hashicorp.consul.internal.configentry.HTTPRouteRule.Matches:type_name -> hashicorp.consul.internal.configentry.HTTPMatch - 68, // 98: hashicorp.consul.internal.configentry.HTTPRouteRule.Services:type_name -> hashicorp.consul.internal.configentry.HTTPService - 62, // 99: hashicorp.consul.internal.configentry.HTTPMatch.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatch - 6, // 100: hashicorp.consul.internal.configentry.HTTPMatch.Method:type_name -> hashicorp.consul.internal.configentry.HTTPMatchMethod - 63, // 101: hashicorp.consul.internal.configentry.HTTPMatch.Path:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatch - 64, // 102: hashicorp.consul.internal.configentry.HTTPMatch.Query:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatch - 7, // 103: hashicorp.consul.internal.configentry.HTTPHeaderMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatchType - 8, // 104: hashicorp.consul.internal.configentry.HTTPPathMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatchType - 9, // 105: hashicorp.consul.internal.configentry.HTTPQueryMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatchType - 67, // 106: hashicorp.consul.internal.configentry.HTTPFilters.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter - 66, // 107: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrites:type_name -> hashicorp.consul.internal.configentry.URLRewrite - 86, // 108: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.AddEntry - 87, // 109: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.SetEntry - 65, // 110: hashicorp.consul.internal.configentry.HTTPService.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters - 89, // 111: hashicorp.consul.internal.configentry.HTTPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 88, // 112: hashicorp.consul.internal.configentry.TCPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.TCPRoute.MetaEntry - 55, // 113: hashicorp.consul.internal.configentry.TCPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 70, // 114: hashicorp.consul.internal.configentry.TCPRoute.Services:type_name -> hashicorp.consul.internal.configentry.TCPService - 51, // 115: hashicorp.consul.internal.configentry.TCPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status - 89, // 116: hashicorp.consul.internal.configentry.TCPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 18, // 117: hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverSubset - 20, // 118: hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailover - 119, // [119:119] is the sub-list for method output_type - 119, // [119:119] is the sub-list for method input_type - 119, // [119:119] is the sub-list for extension type_name - 119, // [119:119] is the sub-list for extension extendee - 0, // [0:119] is the sub-list for field type_name + 58, // 12: hashicorp.consul.internal.configentry.ConfigEntry.InlineCertificate:type_name -> hashicorp.consul.internal.configentry.InlineCertificate + 12, // 13: hashicorp.consul.internal.configentry.MeshConfig.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyMeshConfig + 13, // 14: hashicorp.consul.internal.configentry.MeshConfig.TLS:type_name -> hashicorp.consul.internal.configentry.MeshTLSConfig + 15, // 15: hashicorp.consul.internal.configentry.MeshConfig.HTTP:type_name -> hashicorp.consul.internal.configentry.MeshHTTPConfig + 71, // 16: hashicorp.consul.internal.configentry.MeshConfig.Meta:type_name -> hashicorp.consul.internal.configentry.MeshConfig.MetaEntry + 16, // 17: hashicorp.consul.internal.configentry.MeshConfig.Peering:type_name -> hashicorp.consul.internal.configentry.PeeringMeshConfig + 14, // 18: hashicorp.consul.internal.configentry.MeshTLSConfig.Incoming:type_name -> hashicorp.consul.internal.configentry.MeshDirectionalTLSConfig + 14, // 19: hashicorp.consul.internal.configentry.MeshTLSConfig.Outgoing:type_name -> hashicorp.consul.internal.configentry.MeshDirectionalTLSConfig + 72, // 20: hashicorp.consul.internal.configentry.ServiceResolver.Subsets:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry + 19, // 21: hashicorp.consul.internal.configentry.ServiceResolver.Redirect:type_name -> hashicorp.consul.internal.configentry.ServiceResolverRedirect + 73, // 22: hashicorp.consul.internal.configentry.ServiceResolver.Failover:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry + 91, // 23: hashicorp.consul.internal.configentry.ServiceResolver.ConnectTimeout:type_name -> google.protobuf.Duration + 22, // 24: hashicorp.consul.internal.configentry.ServiceResolver.LoadBalancer:type_name -> hashicorp.consul.internal.configentry.LoadBalancer + 74, // 25: hashicorp.consul.internal.configentry.ServiceResolver.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.MetaEntry + 21, // 26: hashicorp.consul.internal.configentry.ServiceResolverFailover.Targets:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailoverTarget + 23, // 27: hashicorp.consul.internal.configentry.LoadBalancer.RingHashConfig:type_name -> hashicorp.consul.internal.configentry.RingHashConfig + 24, // 28: hashicorp.consul.internal.configentry.LoadBalancer.LeastRequestConfig:type_name -> hashicorp.consul.internal.configentry.LeastRequestConfig + 25, // 29: hashicorp.consul.internal.configentry.LoadBalancer.HashPolicies:type_name -> hashicorp.consul.internal.configentry.HashPolicy + 26, // 30: hashicorp.consul.internal.configentry.HashPolicy.CookieConfig:type_name -> hashicorp.consul.internal.configentry.CookieConfig + 91, // 31: hashicorp.consul.internal.configentry.CookieConfig.TTL:type_name -> google.protobuf.Duration + 29, // 32: hashicorp.consul.internal.configentry.IngressGateway.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig + 31, // 33: hashicorp.consul.internal.configentry.IngressGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.IngressListener + 75, // 34: hashicorp.consul.internal.configentry.IngressGateway.Meta:type_name -> hashicorp.consul.internal.configentry.IngressGateway.MetaEntry + 28, // 35: hashicorp.consul.internal.configentry.IngressGateway.Defaults:type_name -> hashicorp.consul.internal.configentry.IngressServiceConfig + 48, // 36: hashicorp.consul.internal.configentry.IngressServiceConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 30, // 37: hashicorp.consul.internal.configentry.GatewayTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig + 32, // 38: hashicorp.consul.internal.configentry.IngressListener.Services:type_name -> hashicorp.consul.internal.configentry.IngressService + 29, // 39: hashicorp.consul.internal.configentry.IngressListener.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig + 33, // 40: hashicorp.consul.internal.configentry.IngressService.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayServiceTLSConfig + 34, // 41: hashicorp.consul.internal.configentry.IngressService.RequestHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers + 34, // 42: hashicorp.consul.internal.configentry.IngressService.ResponseHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers + 76, // 43: hashicorp.consul.internal.configentry.IngressService.Meta:type_name -> hashicorp.consul.internal.configentry.IngressService.MetaEntry + 89, // 44: hashicorp.consul.internal.configentry.IngressService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 48, // 45: hashicorp.consul.internal.configentry.IngressService.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 30, // 46: hashicorp.consul.internal.configentry.GatewayServiceTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig + 77, // 47: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.AddEntry + 78, // 48: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.SetEntry + 36, // 49: hashicorp.consul.internal.configentry.ServiceIntentions.Sources:type_name -> hashicorp.consul.internal.configentry.SourceIntention + 79, // 50: hashicorp.consul.internal.configentry.ServiceIntentions.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceIntentions.MetaEntry + 1, // 51: hashicorp.consul.internal.configentry.SourceIntention.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction + 37, // 52: hashicorp.consul.internal.configentry.SourceIntention.Permissions:type_name -> hashicorp.consul.internal.configentry.IntentionPermission + 2, // 53: hashicorp.consul.internal.configentry.SourceIntention.Type:type_name -> hashicorp.consul.internal.configentry.IntentionSourceType + 80, // 54: hashicorp.consul.internal.configentry.SourceIntention.LegacyMeta:type_name -> hashicorp.consul.internal.configentry.SourceIntention.LegacyMetaEntry + 92, // 55: hashicorp.consul.internal.configentry.SourceIntention.LegacyCreateTime:type_name -> google.protobuf.Timestamp + 92, // 56: hashicorp.consul.internal.configentry.SourceIntention.LegacyUpdateTime:type_name -> google.protobuf.Timestamp + 89, // 57: hashicorp.consul.internal.configentry.SourceIntention.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 1, // 58: hashicorp.consul.internal.configentry.IntentionPermission.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction + 38, // 59: hashicorp.consul.internal.configentry.IntentionPermission.HTTP:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPPermission + 39, // 60: hashicorp.consul.internal.configentry.IntentionHTTPPermission.Header:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPHeaderPermission + 3, // 61: hashicorp.consul.internal.configentry.ServiceDefaults.Mode:type_name -> hashicorp.consul.internal.configentry.ProxyMode + 41, // 62: hashicorp.consul.internal.configentry.ServiceDefaults.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyConfig + 42, // 63: hashicorp.consul.internal.configentry.ServiceDefaults.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig + 43, // 64: hashicorp.consul.internal.configentry.ServiceDefaults.Expose:type_name -> hashicorp.consul.internal.configentry.ExposeConfig + 45, // 65: hashicorp.consul.internal.configentry.ServiceDefaults.UpstreamConfig:type_name -> hashicorp.consul.internal.configentry.UpstreamConfiguration + 49, // 66: hashicorp.consul.internal.configentry.ServiceDefaults.Destination:type_name -> hashicorp.consul.internal.configentry.DestinationConfig + 81, // 67: hashicorp.consul.internal.configentry.ServiceDefaults.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceDefaults.MetaEntry + 93, // 68: hashicorp.consul.internal.configentry.ServiceDefaults.EnvoyExtensions:type_name -> hashicorp.consul.internal.common.EnvoyExtension + 4, // 69: hashicorp.consul.internal.configentry.MeshGatewayConfig.Mode:type_name -> hashicorp.consul.internal.configentry.MeshGatewayMode + 44, // 70: hashicorp.consul.internal.configentry.ExposeConfig.Paths:type_name -> hashicorp.consul.internal.configentry.ExposePath + 46, // 71: hashicorp.consul.internal.configentry.UpstreamConfiguration.Overrides:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig + 46, // 72: hashicorp.consul.internal.configentry.UpstreamConfiguration.Defaults:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig + 89, // 73: hashicorp.consul.internal.configentry.UpstreamConfig.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 47, // 74: hashicorp.consul.internal.configentry.UpstreamConfig.Limits:type_name -> hashicorp.consul.internal.configentry.UpstreamLimits + 48, // 75: hashicorp.consul.internal.configentry.UpstreamConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 42, // 76: hashicorp.consul.internal.configentry.UpstreamConfig.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig + 91, // 77: hashicorp.consul.internal.configentry.PassiveHealthCheck.Interval:type_name -> google.protobuf.Duration + 82, // 78: hashicorp.consul.internal.configentry.APIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.APIGateway.MetaEntry + 53, // 79: hashicorp.consul.internal.configentry.APIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.APIGatewayListener + 51, // 80: hashicorp.consul.internal.configentry.APIGateway.Status:type_name -> hashicorp.consul.internal.configentry.Status + 52, // 81: hashicorp.consul.internal.configentry.Status.Conditions:type_name -> hashicorp.consul.internal.configentry.Condition + 55, // 82: hashicorp.consul.internal.configentry.Condition.Resource:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 92, // 83: hashicorp.consul.internal.configentry.Condition.LastTransitionTime:type_name -> google.protobuf.Timestamp + 5, // 84: hashicorp.consul.internal.configentry.APIGatewayListener.Protocol:type_name -> hashicorp.consul.internal.configentry.APIGatewayListenerProtocol + 54, // 85: hashicorp.consul.internal.configentry.APIGatewayListener.TLS:type_name -> hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration + 55, // 86: hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 89, // 87: hashicorp.consul.internal.configentry.ResourceReference.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 83, // 88: hashicorp.consul.internal.configentry.BoundAPIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.BoundAPIGateway.MetaEntry + 57, // 89: hashicorp.consul.internal.configentry.BoundAPIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.BoundAPIGatewayListener + 55, // 90: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 55, // 91: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Routes:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 84, // 92: hashicorp.consul.internal.configentry.InlineCertificate.Meta:type_name -> hashicorp.consul.internal.configentry.InlineCertificate.MetaEntry + 85, // 93: hashicorp.consul.internal.configentry.HTTPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.HTTPRoute.MetaEntry + 55, // 94: hashicorp.consul.internal.configentry.HTTPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 60, // 95: hashicorp.consul.internal.configentry.HTTPRoute.Rules:type_name -> hashicorp.consul.internal.configentry.HTTPRouteRule + 51, // 96: hashicorp.consul.internal.configentry.HTTPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status + 65, // 97: hashicorp.consul.internal.configentry.HTTPRouteRule.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters + 61, // 98: hashicorp.consul.internal.configentry.HTTPRouteRule.Matches:type_name -> hashicorp.consul.internal.configentry.HTTPMatch + 68, // 99: hashicorp.consul.internal.configentry.HTTPRouteRule.Services:type_name -> hashicorp.consul.internal.configentry.HTTPService + 62, // 100: hashicorp.consul.internal.configentry.HTTPMatch.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatch + 6, // 101: hashicorp.consul.internal.configentry.HTTPMatch.Method:type_name -> hashicorp.consul.internal.configentry.HTTPMatchMethod + 63, // 102: hashicorp.consul.internal.configentry.HTTPMatch.Path:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatch + 64, // 103: hashicorp.consul.internal.configentry.HTTPMatch.Query:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatch + 7, // 104: hashicorp.consul.internal.configentry.HTTPHeaderMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatchType + 8, // 105: hashicorp.consul.internal.configentry.HTTPPathMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatchType + 9, // 106: hashicorp.consul.internal.configentry.HTTPQueryMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatchType + 67, // 107: hashicorp.consul.internal.configentry.HTTPFilters.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter + 66, // 108: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrites:type_name -> hashicorp.consul.internal.configentry.URLRewrite + 86, // 109: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.AddEntry + 87, // 110: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.SetEntry + 65, // 111: hashicorp.consul.internal.configentry.HTTPService.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters + 89, // 112: hashicorp.consul.internal.configentry.HTTPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 88, // 113: hashicorp.consul.internal.configentry.TCPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.TCPRoute.MetaEntry + 55, // 114: hashicorp.consul.internal.configentry.TCPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 70, // 115: hashicorp.consul.internal.configentry.TCPRoute.Services:type_name -> hashicorp.consul.internal.configentry.TCPService + 51, // 116: hashicorp.consul.internal.configentry.TCPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status + 89, // 117: hashicorp.consul.internal.configentry.TCPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 18, // 118: hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverSubset + 20, // 119: hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailover + 120, // [120:120] is the sub-list for method output_type + 120, // [120:120] is the sub-list for method input_type + 120, // [120:120] is the sub-list for extension type_name + 120, // [120:120] is the sub-list for extension extendee + 0, // [0:120] is the sub-list for field type_name } func init() { file_proto_pbconfigentry_config_entry_proto_init() } @@ -7426,6 +7448,7 @@ func file_proto_pbconfigentry_config_entry_proto_init() { (*ConfigEntry_BoundAPIGateway)(nil), (*ConfigEntry_TCPRoute)(nil), (*ConfigEntry_HTTPRoute)(nil), + (*ConfigEntry_InlineCertificate)(nil), } type x struct{} out := protoimpl.TypeBuilder{ diff --git a/proto/pbconfigentry/config_entry.proto b/proto/pbconfigentry/config_entry.proto index 53d69a64bc77..6749d5838c5d 100644 --- a/proto/pbconfigentry/config_entry.proto +++ b/proto/pbconfigentry/config_entry.proto @@ -37,6 +37,7 @@ message ConfigEntry { BoundAPIGateway BoundAPIGateway = 11; TCPRoute TCPRoute = 12; HTTPRoute HTTPRoute = 13; + InlineCertificate InlineCertificate = 14; } } From 9aa85df0ccbc56a38e7776cda5f9a731de283353 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Feb 2023 13:17:15 -0500 Subject: [PATCH 003/421] backport of commit e5adc43315a5e35c522fcf221ec96ebdc4113664 (#16279) Co-authored-by: Derek Menteer --- agent/proxycfg/api_gateway.go | 1 + agent/proxycfg/ingress_gateway.go | 1 + agent/proxycfg/manager.go | 15 +++++++++++++++ agent/proxycfg/state.go | 16 ++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/agent/proxycfg/api_gateway.go b/agent/proxycfg/api_gateway.go index 28a6c4230066..c9b5ad1007e8 100644 --- a/agent/proxycfg/api_gateway.go +++ b/agent/proxycfg/api_gateway.go @@ -73,6 +73,7 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err snap.APIGateway.WatchedDiscoveryChains = make(map[UpstreamID]context.CancelFunc) snap.APIGateway.WatchedGateways = make(map[UpstreamID]map[string]context.CancelFunc) snap.APIGateway.WatchedGatewayEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes) + snap.APIGateway.WatchedLocalGWEndpoints = watch.NewMap[string, structs.CheckServiceNodes]() snap.APIGateway.WatchedUpstreams = make(map[UpstreamID]map[string]context.CancelFunc) snap.APIGateway.WatchedUpstreamEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes) diff --git a/agent/proxycfg/ingress_gateway.go b/agent/proxycfg/ingress_gateway.go index b6c1cd6e0297..f0f75d8107c3 100644 --- a/agent/proxycfg/ingress_gateway.go +++ b/agent/proxycfg/ingress_gateway.go @@ -67,6 +67,7 @@ func (s *handlerIngressGateway) initialize(ctx context.Context) (ConfigSnapshot, snap.IngressGateway.WatchedUpstreamEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes) snap.IngressGateway.WatchedGateways = make(map[UpstreamID]map[string]context.CancelFunc) snap.IngressGateway.WatchedGatewayEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes) + snap.IngressGateway.WatchedLocalGWEndpoints = watch.NewMap[string, structs.CheckServiceNodes]() snap.IngressGateway.Listeners = make(map[IngressListenerKey]structs.IngressListener) snap.IngressGateway.UpstreamPeerTrustBundles = watch.NewMap[string, *pbpeering.PeeringTrustBundle]() snap.IngressGateway.PeerUpstreamEndpoints = watch.NewMap[UpstreamID, structs.CheckServiceNodes]() diff --git a/agent/proxycfg/manager.go b/agent/proxycfg/manager.go index eb5df5a85521..c58268e7e039 100644 --- a/agent/proxycfg/manager.go +++ b/agent/proxycfg/manager.go @@ -2,6 +2,7 @@ package proxycfg import ( "errors" + "runtime/debug" "sync" "github.com/hashicorp/go-hclog" @@ -142,6 +143,20 @@ func (m *Manager) Register(id ProxyID, ns *structs.NodeService, source ProxySour m.mu.Lock() defer m.mu.Unlock() + defer func() { + if r := recover(); r != nil { + m.Logger.Error("unexpected panic during service manager registration", + "node", id.NodeName, + "service", id.ServiceID, + "message", r, + "stacktrace", string(debug.Stack()), + ) + } + }() + return m.register(id, ns, source, token, overwrite) +} + +func (m *Manager) register(id ProxyID, ns *structs.NodeService, source ProxySource, token string, overwrite bool) error { state, ok := m.proxies[id] if ok { if state.source != source && !overwrite { diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 57964d54fe16..d312c3b4c10c 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "reflect" + "runtime/debug" "sync/atomic" "time" @@ -298,6 +299,21 @@ func newConfigSnapshotFromServiceInstance(s serviceInstance, config stateConfig) } func (s *state) run(ctx context.Context, snap *ConfigSnapshot) { + // Add a recover here so than any panics do not make their way up + // into the server / agent. + defer func() { + if r := recover(); r != nil { + s.logger.Error("unexpected panic while running proxycfg", + "node", s.serviceInstance.proxyID.NodeName, + "service", s.serviceInstance.proxyID.ServiceID, + "message", r, + "stacktrace", string(debug.Stack())) + } + }() + s.unsafeRun(ctx, snap) +} + +func (s *state) unsafeRun(ctx context.Context, snap *ConfigSnapshot) { // Close the channel we return from Watch when we stop so consumers can stop // watching and clean up their goroutines. It's important we do this here and // not in Close since this routine sends on this chan and so might panic if it From 0a32425202baf1a99d07156c2cebc054a40fb879 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Feb 2023 15:03:29 -0500 Subject: [PATCH 004/421] Backport of Fix infinite recursion in inline-certificate config entry into release/1.15.x (#16281) * backport of commit ea4dd2bb4cefb03593573cfec1a202fb98468a44 * backport of commit 637d13a0a915baa72d769bcf5d92ec6279631ac1 * backport of commit 18f19cecd18f2935e45fe0132763982c02a3eaaf * backport of commit 3e6807bdbc9eadbb81e9c30788eeac573b0d17a8 --------- Co-authored-by: Nathan Coleman --- .../config_entry_inline_certificate.go | 38 +----- api/config_entry_inline_certificate.go | 48 ++----- api/config_entry_inline_certificate_test.go | 122 ++++++++++++++++++ 3 files changed, 137 insertions(+), 71 deletions(-) create mode 100644 api/config_entry_inline_certificate_test.go diff --git a/agent/structs/config_entry_inline_certificate.go b/agent/structs/config_entry_inline_certificate.go index 7cfc71a6b2c7..24028ffff2fa 100644 --- a/agent/structs/config_entry_inline_certificate.go +++ b/agent/structs/config_entry_inline_certificate.go @@ -29,17 +29,14 @@ type InlineCertificateConfigEntry struct { RaftIndex } -func (e *InlineCertificateConfigEntry) GetKind() string { - return InlineCertificate -} - -func (e *InlineCertificateConfigEntry) GetName() string { - return e.Name -} - -func (e *InlineCertificateConfigEntry) Normalize() error { - return nil +func (e *InlineCertificateConfigEntry) GetKind() string { return InlineCertificate } +func (e *InlineCertificateConfigEntry) GetName() string { return e.Name } +func (e *InlineCertificateConfigEntry) Normalize() error { return nil } +func (e *InlineCertificateConfigEntry) GetMeta() map[string]string { return e.Meta } +func (e *InlineCertificateConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { + return &e.EnterpriseMeta } +func (e *InlineCertificateConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } func (e *InlineCertificateConfigEntry) Validate() error { privateKeyBlock, _ := pem.Decode([]byte(e.PrivateKey)) @@ -78,24 +75,3 @@ func (e *InlineCertificateConfigEntry) CanWrite(authz acl.Authorizer) error { e.FillAuthzContext(&authzContext) return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext) } - -func (e *InlineCertificateConfigEntry) GetMeta() map[string]string { - if e == nil { - return nil - } - return e.Meta -} - -func (e *InlineCertificateConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { - if e == nil { - return nil - } - return &e.EnterpriseMeta -} - -func (e *InlineCertificateConfigEntry) GetRaftIndex() *RaftIndex { - if e == nil { - return &RaftIndex{} - } - return &e.RaftIndex -} diff --git a/api/config_entry_inline_certificate.go b/api/config_entry_inline_certificate.go index d2aa5115eac5..bbf12ccaaf69 100644 --- a/api/config_entry_inline_certificate.go +++ b/api/config_entry_inline_certificate.go @@ -12,7 +12,7 @@ type InlineCertificateConfigEntry struct { // Certificate is the public certificate component of an x509 key pair encoded in raw PEM format. Certificate string // PrivateKey is the private key component of an x509 key pair encoded in raw PEM format. - PrivateKey string + PrivateKey string `alias:"private_key"` Meta map[string]string `json:",omitempty"` @@ -34,42 +34,10 @@ type InlineCertificateConfigEntry struct { Namespace string `json:",omitempty"` } -func (a *InlineCertificateConfigEntry) GetKind() string { - return InlineCertificate -} - -func (a *InlineCertificateConfigEntry) GetName() string { - if a != nil { - return "" - } - return a.Name -} - -func (a *InlineCertificateConfigEntry) GetPartition() string { - if a != nil { - return "" - } - return a.Partition -} - -func (a *InlineCertificateConfigEntry) GetNamespace() string { - if a != nil { - return "" - } - return a.GetNamespace() -} - -func (a *InlineCertificateConfigEntry) GetMeta() map[string]string { - if a != nil { - return nil - } - return a.GetMeta() -} - -func (a *InlineCertificateConfigEntry) GetCreateIndex() uint64 { - return a.CreateIndex -} - -func (a *InlineCertificateConfigEntry) GetModifyIndex() uint64 { - return a.ModifyIndex -} +func (a *InlineCertificateConfigEntry) GetKind() string { return InlineCertificate } +func (a *InlineCertificateConfigEntry) GetName() string { return a.Name } +func (a *InlineCertificateConfigEntry) GetPartition() string { return a.Partition } +func (a *InlineCertificateConfigEntry) GetNamespace() string { return a.Namespace } +func (a *InlineCertificateConfigEntry) GetMeta() map[string]string { return a.Meta } +func (a *InlineCertificateConfigEntry) GetCreateIndex() uint64 { return a.CreateIndex } +func (a *InlineCertificateConfigEntry) GetModifyIndex() uint64 { return a.ModifyIndex } diff --git a/api/config_entry_inline_certificate_test.go b/api/config_entry_inline_certificate_test.go new file mode 100644 index 000000000000..3b1cc1f54ef6 --- /dev/null +++ b/api/config_entry_inline_certificate_test.go @@ -0,0 +1,122 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // generated via openssl req -x509 -sha256 -days 1825 -newkey rsa:2048 -keyout private.key -out certificate.crt + validPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAx95Opa6t4lGEpiTUogEBptqOdam2ch4BHQGhNhX/MrDwwuZQ +httBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2jQlhqTodElkbsd5vWY8R/bxJWQSo +NvVE12TlzECxGpJEiHt4W0r8pGffk+rvpljiUyCfnT1kGF3znOSjK1hRMTn6RKWC +yYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409g9X5VU88/Bmmrz4cMyxce86Kc2ug +5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftrXOvuCbO5IBRHMOBHiHTZ4rtGuhMa +Ir21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+WmQIDAQABAoIBACYvceUzp2MK4gYA +GWPOP2uKbBdM0l+hHeNV0WAM+dHMfmMuL4pkT36ucqt0ySOLjw6rQyOZG5nmA6t9 +sv0g4ae2eCMlyDIeNi1Yavu4Wt6YX4cTXbQKThm83C6W2X9THKbauBbxD621bsDK +7PhiGPN60yPue7YwFQAPqqD4YaK+s22HFIzk9gwM/rkvAUNwRv7SyHMiFe4Igc1C +Eev7iHWzvj5Heoz6XfF+XNF9DU+TieSUAdjd56VyUb8XL4+uBTOhHwLiXvAmfaMR +HvpcxeKnYZusS6NaOxcUHiJnsLNWrxmJj9WEGgQzuLxcLjTe4vVmELVZD8t3QUKj +PAxu8tUCgYEA7KIWVn9dfVpokReorFym+J8FzLwSktP9RZYEMonJo00i8aii3K9s +u/aSwRWQSCzmON1ZcxZzWhwQF9usz6kGCk//9+4hlVW90GtNK0RD+j7sp4aT2JI8 +9eLEjTG+xSXa7XWe98QncjjL9lu/yrRncSTxHs13q/XP198nn2aYuQ8CgYEA2Dnt +sRBzv0fFEvzzFv7G/5f85mouN38TUYvxNRTjBLCXl9DeKjDkOVZ2b6qlfQnYXIru +H+W+v+AZEb6fySXc8FRab7lkgTMrwE+aeI4rkW7asVwtclv01QJ5wMnyT84AgDD/ +Dgt/RThFaHgtU9TW5GOZveL+l9fVPn7vKFdTJdcCgYEArJ99zjHxwJ1whNAOk1av +09UmRPm6TvRo4heTDk8oEoIWCNatoHI0z1YMLuENNSnT9Q280FFDayvnrY/qnD7A +kktT/sjwJOG8q8trKzIMqQS4XWm2dxoPcIyyOBJfCbEY6XuRsUuePxwh5qF942EB +yS9a2s6nC4Ix0lgPrqAIr48CgYBgS/Q6riwOXSU8nqCYdiEkBYlhCJrKpnJxF9T1 +ofa0yPzKZP/8ZEfP7VzTwHjxJehQ1qLUW9pG08P2biH1UEKEWdzo8vT6wVJT1F/k +HtTycR8+a+Hlk2SHVRHqNUYQGpuIe8mrdJ1as4Pd0d/F/P0zO9Rlh+mAsGPM8HUM +T0+9gwKBgHDoerX7NTskg0H0t8O+iSMevdxpEWp34ZYa9gHiftTQGyrRgERCa7Gj +nZPAxKb2JoWyfnu3v7G5gZ8fhDFsiOxLbZv6UZJBbUIh1MjJISpXrForDrC2QNLX +kHrHfwBFDB3KMudhQknsJzEJKCL/KmFH6o0MvsoaT9yzEl3K+ah/ +-----END RSA PRIVATE KEY-----` + validCertificate = `-----BEGIN CERTIFICATE----- +MIICljCCAX4CCQCQMDsYO8FrPjANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV +UzAeFw0yMjEyMjAxNzUwMjVaFw0yNzEyMTkxNzUwMjVaMA0xCzAJBgNVBAYTAlVT +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx95Opa6t4lGEpiTUogEB +ptqOdam2ch4BHQGhNhX/MrDwwuZQhttBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2 +jQlhqTodElkbsd5vWY8R/bxJWQSoNvVE12TlzECxGpJEiHt4W0r8pGffk+rvplji +UyCfnT1kGF3znOSjK1hRMTn6RKWCyYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409 +g9X5VU88/Bmmrz4cMyxce86Kc2ug5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftr +XOvuCbO5IBRHMOBHiHTZ4rtGuhMaIr21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+W +mQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBfCqoUIdPf/HGSbOorPyZWbyizNtHJ +GL7x9cAeIYxpI5Y/WcO1o5v94lvrgm3FNfJoGKbV66+JxOge731FrfMpHplhar1Z +RahYIzNLRBTLrwadLAZkApUpZvB8qDK4knsTWFYujNsylCww2A6ajzIMFNU4GkUK +NtyHRuD+KYRmjXtyX1yHNqfGN3vOQmwavHq2R8wHYuBSc6LAHHV9vG+j0VsgMELO +qwxn8SmLkSKbf2+MsQVzLCXXN5u+D8Yv+4py+oKP4EQ5aFZuDEx+r/G/31rTthww +AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r +-----END CERTIFICATE-----` +) + +func TestAPI_ConfigEntries_InlineCertificate(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + configEntries := c.ConfigEntries() + + cert1 := &InlineCertificateConfigEntry{ + Kind: InlineCertificate, + Name: "cert1", + Meta: map[string]string{"foo": "bar"}, + Certificate: validCertificate, + PrivateKey: validPrivateKey, + } + + // set it + _, wm, err := configEntries.Set(cert1, nil) + require.NoError(t, err) + assert.NotNil(t, wm) + + // get it + entry, qm, err := configEntries.Get(InlineCertificate, "cert1", nil) + require.NoError(t, err) + require.NotNil(t, qm) + assert.NotEqual(t, 0, qm.RequestTime) + + readCert, ok := entry.(*InlineCertificateConfigEntry) + require.True(t, ok) + assert.Equal(t, cert1.Kind, readCert.Kind) + assert.Equal(t, cert1.Name, readCert.Name) + assert.Equal(t, cert1.Meta, readCert.Meta) + assert.Equal(t, cert1.Meta, readCert.GetMeta()) + + // update it + cert1.Meta["bar"] = "baz" + written, wm, err := configEntries.CAS(cert1, readCert.ModifyIndex, nil) + require.NoError(t, err) + require.NotNil(t, wm) + assert.NotEqual(t, 0, wm.RequestTime) + assert.True(t, written) + + // list it + entries, qm, err := configEntries.List(InlineCertificate, nil) + require.NoError(t, err) + require.NotNil(t, qm) + assert.NotEqual(t, 0, qm.RequestTime) + + require.Len(t, entries, 1) + assert.Equal(t, cert1.Kind, entries[0].GetKind()) + assert.Equal(t, cert1.Name, entries[0].GetName()) + + readCert, ok = entries[0].(*InlineCertificateConfigEntry) + require.True(t, ok) + assert.Equal(t, cert1.Certificate, readCert.Certificate) + assert.Equal(t, cert1.Meta, readCert.Meta) + + // delete it + wm, err = configEntries.Delete(InlineCertificate, cert1.Name, nil) + require.NoError(t, err) + require.NotNil(t, wm) + assert.NotEqual(t, 0, wm.RequestTime) + + // try to get it + _, _, err = configEntries.Get(InlineCertificate, cert1.Name, nil) + assert.Error(t, err) +} From 6cd08b96c33c3af5713bec096483ed5a552551fd Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 16 Feb 2023 10:51:57 -0500 Subject: [PATCH 005/421] Backport of Fix mesh gateways incorrectly matching peer locality. into release/1.15.x (#16287) * backport of commit 9bc7f6280a8a81eb7029731f4d7364c9f71911d5 * backport of commit 0c48c9a1e8c103fd76753bab0897dce1f594d85f --------- Co-authored-by: Derek Menteer --- .changelog/16257.txt | 3 +++ agent/proxycfg/testing_mesh_gateway.go | 8 ++++---- agent/proxycfg/testing_upstreams.go | 4 ++-- agent/structs/testing_catalog.go | 27 ++++++++++++++++++-------- agent/xds/endpoints.go | 4 +++- 5 files changed, 31 insertions(+), 15 deletions(-) create mode 100644 .changelog/16257.txt diff --git a/.changelog/16257.txt b/.changelog/16257.txt new file mode 100644 index 000000000000..8e98530c421a --- /dev/null +++ b/.changelog/16257.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue where mesh gateways would use the wrong address when contacting a remote peer with the same datacenter name. +``` diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go index 039807ed6199..f6b463f9d3fc 100644 --- a/agent/proxycfg/testing_mesh_gateway.go +++ b/agent/proxycfg/testing_mesh_gateway.go @@ -659,8 +659,8 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( CorrelationID: "peering-connect-service:peer-a:db", Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{ - structs.TestCheckNodeServiceWithNameInPeer(t, "db", "peer-a", "10.40.1.1", false), - structs.TestCheckNodeServiceWithNameInPeer(t, "db", "peer-a", "10.40.1.2", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.1", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.2", false), }, }, }, @@ -668,8 +668,8 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( CorrelationID: "peering-connect-service:peer-b:alt", Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{ - structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "peer-b", "10.40.2.1", false), - structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "peer-b", "10.40.2.2", true), + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.1", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.2", true), }, }, }, diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index ed2e65d8d678..cb38a3de16ad 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -94,7 +94,7 @@ func setupTestVariationConfigEntriesAndSnapshot( events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:db?peer=cluster-01", Result: &structs.IndexedCheckServiceNodes{ - Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "cluster-01", "10.40.1.1", false)}, + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "cluster-01", "10.40.1.1", false)}, }, }) case "redirect-to-cluster-peer": @@ -112,7 +112,7 @@ func setupTestVariationConfigEntriesAndSnapshot( events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:db?peer=cluster-01", Result: &structs.IndexedCheckServiceNodes{ - Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "cluster-01", "10.40.1.1", false)}, + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc2", "cluster-01", "10.40.1.1", false)}, }, }) case "failover-through-double-remote-gateway-triggered": diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index 11316905117f..f7f2a7e710e8 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -55,11 +55,13 @@ func TestNodeServiceWithName(t testing.T, name string) *NodeService { const peerTrustDomain = "1c053652-8512-4373-90cf-5a7f6263a994.consul" -func TestCheckNodeServiceWithNameInPeer(t testing.T, name, peer, ip string, useHostname bool) CheckServiceNode { +func TestCheckNodeServiceWithNameInPeer(t testing.T, name, dc, peer, ip string, useHostname bool) CheckServiceNode { service := &NodeService{ - Kind: ServiceKindTypical, - Service: name, - Port: 8080, + Kind: ServiceKindTypical, + Service: name, + // We should not see this port number appear in most xds golden tests, + // because the WAN addr should typically be used. + Port: 9090, PeerName: peer, Connect: ServiceConnect{ PeerMeta: &PeeringServiceMeta{ @@ -72,6 +74,13 @@ func TestCheckNodeServiceWithNameInPeer(t testing.T, name, peer, ip string, useH Protocol: "tcp", }, }, + // This value should typically be seen in golden file output, since this is a peered service. + TaggedAddresses: map[string]ServiceAddress{ + TaggedAddressWAN: { + Address: ip, + Port: 8080, + }, + }, } if useHostname { @@ -89,10 +98,12 @@ func TestCheckNodeServiceWithNameInPeer(t testing.T, name, peer, ip string, useH return CheckServiceNode{ Node: &Node{ - ID: "test1", - Node: "test1", - Address: ip, - Datacenter: "cloud-dc", + ID: "test1", + Node: "test1", + // We should not see this address appear in most xds golden tests, + // because the WAN addr should typically be used. + Address: "1.23.45.67", + Datacenter: dc, }, Service: service, } diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index dcdbb4d2f5d7..d117f8dd8298 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -449,7 +449,9 @@ func (s *ResourceGenerator) makeEndpointsForOutgoingPeeredServices( la := makeLoadAssignment( clusterName, groups, - cfgSnap.Locality, + // Use an empty key here so that it never matches. This will force the mesh gateway to always + // reference the remote mesh gateway's wan addr. + proxycfg.GatewayKey{}, ) resources = append(resources, la) } From 94f4347ffebbf6fb0be4dc97aae22cf4684bd0e1 Mon Sep 17 00:00:00 2001 From: Nitya Dhanushkodi Date: Fri, 17 Feb 2023 10:53:20 -0800 Subject: [PATCH 006/421] Backport: troubleshoot: fixes and updated messages (#16294) (#16309) --- .../validateupstream_test.go | 12 +++-- .../troubleshoot/proxy/troubleshoot_proxy.go | 20 ++++--- .../upstreams/troubleshoot_upstreams.go | 16 +++--- .../troubleshoot_upstream_test.go | 25 +++++---- troubleshoot/proxy/certs.go | 20 +++++-- troubleshoot/proxy/certs_test.go | 12 +++-- troubleshoot/proxy/stats.go | 20 +++++-- troubleshoot/proxy/troubleshoot_proxy.go | 13 +---- troubleshoot/proxy/upstreams.go | 5 +- troubleshoot/proxy/upstreams_test.go | 9 ++-- troubleshoot/validate/validate.go | 53 ++++++++++++++----- troubleshoot/validate/validate_test.go | 10 ++-- 12 files changed, 135 insertions(+), 80 deletions(-) diff --git a/agent/xds/validateupstream-test/validateupstream_test.go b/agent/xds/validateupstream-test/validateupstream_test.go index c78b34d7aa05..250b6acdec3b 100644 --- a/agent/xds/validateupstream-test/validateupstream_test.go +++ b/agent/xds/validateupstream-test/validateupstream_test.go @@ -55,7 +55,7 @@ func TestValidateUpstreams(t *testing.T) { delete(ir.Index[xdscommon.ListenerType], listenerName) return ir }, - err: "no listener for upstream \"db\"", + err: "No listener for upstream \"db\"", }, { name: "tcp-missing-cluster", @@ -66,7 +66,7 @@ func TestValidateUpstreams(t *testing.T) { delete(ir.Index[xdscommon.ClusterType], sni) return ir }, - err: "no cluster \"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul\" for upstream \"db\"", + err: "No cluster \"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul\" for upstream \"db\"", }, { name: "http-success", @@ -124,7 +124,7 @@ func TestValidateUpstreams(t *testing.T) { delete(ir.Index[xdscommon.RouteType], "db") return ir }, - err: "no route for upstream \"db\"", + err: "No route for upstream \"db\"", }, { name: "redirect", @@ -170,7 +170,7 @@ func TestValidateUpstreams(t *testing.T) { delete(ir.Index[xdscommon.ClusterType], sni) return ir }, - err: "no cluster \"google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul\" for upstream \"240.0.0.1\"", + err: "No cluster \"google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul\" for upstream \"240.0.0.1\"", }, { name: "tproxy-http-redirect-success", @@ -230,7 +230,9 @@ func TestValidateUpstreams(t *testing.T) { var outputErrors string for _, msgError := range messages.Errors() { outputErrors += msgError.Message - outputErrors += msgError.PossibleActions + for _, action := range msgError.PossibleActions { + outputErrors += action + } } if len(tt.err) == 0 { require.True(t, messages.Success()) diff --git a/command/troubleshoot/proxy/troubleshoot_proxy.go b/command/troubleshoot/proxy/troubleshoot_proxy.go index 8950424a786e..57d982dea0db 100644 --- a/command/troubleshoot/proxy/troubleshoot_proxy.go +++ b/command/troubleshoot/proxy/troubleshoot_proxy.go @@ -77,12 +77,12 @@ func (c *cmd) Run(args []string) int { t, err := troubleshoot.NewTroubleshoot(adminBindIP, adminPort) if err != nil { - c.UI.Error("error generating troubleshoot client: " + err.Error()) + c.UI.Error("Error generating troubleshoot client: " + err.Error()) return 1 } messages, err := t.RunAllTests(c.upstreamEnvoyID, c.upstreamIP) if err != nil { - c.UI.Error("error running the tests: " + err.Error()) + c.UI.Error("Error running the tests: " + err.Error()) return 1 } @@ -92,11 +92,16 @@ func (c *cmd) Run(args []string) int { c.UI.SuccessOutput(o.Message) } else { c.UI.ErrorOutput(o.Message) - if o.PossibleActions != "" { - c.UI.UnchangedOutput(o.PossibleActions) + for _, action := range o.PossibleActions { + c.UI.UnchangedOutput("-> " + action) } } } + if messages.Success() { + c.UI.UnchangedOutput("If you are still experiencing issues, you can:") + c.UI.UnchangedOutput("-> Check intentions to ensure the upstream allows traffic from this source") + c.UI.UnchangedOutput("-> If using transparent proxy, ensure DNS resolution is to the same IP you have verified here") + } return 0 } @@ -114,14 +119,15 @@ const ( Usage: consul troubleshoot proxy [options] Connects to local envoy proxy and troubleshoots service mesh communication issues. - Requires an upstream service envoy identifier. + Requires an upstream service identifier. When debugging explicitly configured upstreams, + use -upstream-envoy-id, when debugging transparent proxy upstreams use -upstream-ip. Examples: (explicit upstreams only) $ consul troubleshoot proxy -upstream-envoy-id foo (transparent proxy only) - $ consul troubleshoot proxy -upstream-ip + $ consul troubleshoot proxy -upstream-ip 240.0.0.1 - where 'foo' is the upstream envoy identifier which + where 'foo' is the upstream envoy identifier and '240.0.0.1' is an upstream ip which can be obtained by running: $ consul troubleshoot upstreams [options] ` diff --git a/command/troubleshoot/upstreams/troubleshoot_upstreams.go b/command/troubleshoot/upstreams/troubleshoot_upstreams.go index 1249f5ddc56d..435630cdc9e9 100644 --- a/command/troubleshoot/upstreams/troubleshoot_upstreams.go +++ b/command/troubleshoot/upstreams/troubleshoot_upstreams.go @@ -77,24 +77,24 @@ func (c *cmd) Run(args []string) int { return 1 } - c.UI.Output(fmt.Sprintf("==> Upstreams (explicit upstreams only) (%v)", len(envoyIDs))) + c.UI.HeaderOutput(fmt.Sprintf("Upstreams (explicit upstreams only) (%v)\n", len(envoyIDs))) for _, u := range envoyIDs { - c.UI.Output(u) + c.UI.UnchangedOutput(u) } - c.UI.Output(fmt.Sprintf("\n==> Upstream IPs (transparent proxy only) (%v)", len(upstreamIPs))) + c.UI.HeaderOutput(fmt.Sprintf("Upstream IPs (transparent proxy only) (%v)", len(upstreamIPs))) tbl := cli.NewTable("IPs ", "Virtual ", "Cluster Names") for _, u := range upstreamIPs { tbl.AddRow([]string{formatIPs(u.IPs), strconv.FormatBool(u.IsVirtual), formatClusterNames(u.ClusterNames)}, []string{}) } c.UI.Table(tbl) - c.UI.Output("\nIf you don't see your upstream address or cluster for a transparent proxy upstream:") - c.UI.Output("- Check intentions: Tproxy upstreams are configured based on intentions, make sure you " + + c.UI.UnchangedOutput("\nIf you cannot find the upstream address or cluster for a transparent proxy upstream:") + c.UI.UnchangedOutput("-> Check intentions: Transparent proxy upstreams are configured based on intentions. Make sure you " + "have configured intentions to allow traffic to your upstream.") - c.UI.Output("- You can also check that the right cluster is being dialed by running a DNS lookup " + - "for the upstream you are dialing (i.e dig backend.svc.consul). If the address you get from that is missing " + - "from the Upstream IPs your proxy may be misconfigured.") + c.UI.UnchangedOutput("-> To check that the right cluster is being dialed, run a DNS lookup " + + "for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing " + + "from the upstream IPs, it means that your proxy may be misconfigured.") return 0 } diff --git a/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go index 7803b38bff4f..3470e738922d 100644 --- a/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go +++ b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go @@ -3,11 +3,12 @@ package troubleshoot import ( "context" "fmt" - "github.com/stretchr/testify/assert" "strings" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" @@ -40,11 +41,12 @@ func TestTroubleshootProxy(t *testing.T) { "-envoy-admin-endpoint", fmt.Sprintf("localhost:%v", clientAdminPort), "-upstream-envoy-id", libservice.StaticServerServiceName}) require.NoError(t, err) - certsValid := strings.Contains(output, "certificates are valid") - listenersExist := strings.Contains(output, fmt.Sprintf("listener for upstream \"%s\" found", libservice.StaticServerServiceName)) - routesExist := strings.Contains(output, fmt.Sprintf("route for upstream \"%s\" found", libservice.StaticServerServiceName)) - healthyEndpoints := strings.Contains(output, "✓ healthy endpoints for cluster") - return upstreamExists && certsValid && listenersExist && routesExist && healthyEndpoints + certsValid := strings.Contains(output, "Certificates are valid") + noRejectedConfig := strings.Contains(output, "Envoy has 0 rejected configurations") + noConnFailure := strings.Contains(output, "Envoy has detected 0 connection failure(s)") + listenersExist := strings.Contains(output, fmt.Sprintf("Listener for upstream \"%s\" found", libservice.StaticServerServiceName)) + healthyEndpoints := strings.Contains(output, "Healthy endpoints for cluster") + return upstreamExists && certsValid && listenersExist && noRejectedConfig && noConnFailure && healthyEndpoints }, 60*time.Second, 10*time.Second) }) @@ -58,11 +60,12 @@ func TestTroubleshootProxy(t *testing.T) { "-upstream-envoy-id", libservice.StaticServerServiceName}) require.NoError(t, err) - certsValid := strings.Contains(output, "certificates are valid") - listenersExist := strings.Contains(output, fmt.Sprintf("listener for upstream \"%s\" found", libservice.StaticServerServiceName)) - routesExist := strings.Contains(output, fmt.Sprintf("route for upstream \"%s\" found", libservice.StaticServerServiceName)) - endpointUnhealthy := strings.Contains(output, "no healthy endpoints for cluster") - return certsValid && listenersExist && routesExist && endpointUnhealthy + certsValid := strings.Contains(output, "Certificates are valid") + noRejectedConfig := strings.Contains(output, "Envoy has 0 rejected configurations") + noConnFailure := strings.Contains(output, "Envoy has detected 0 connection failure(s)") + listenersExist := strings.Contains(output, fmt.Sprintf("Listener for upstream \"%s\" found", libservice.StaticServerServiceName)) + endpointUnhealthy := strings.Contains(output, "No healthy endpoints for cluster") + return certsValid && listenersExist && noRejectedConfig && noConnFailure && endpointUnhealthy }, 60*time.Second, 10*time.Second) }) } diff --git a/troubleshoot/proxy/certs.go b/troubleshoot/proxy/certs.go index 1fa61f3483e5..ec4b2bc703d2 100644 --- a/troubleshoot/proxy/certs.go +++ b/troubleshoot/proxy/certs.go @@ -18,7 +18,10 @@ func (t *Troubleshoot) validateCerts(certs *envoy_admin_v3.Certificates) validat if certs == nil { msg := validate.Message{ Success: false, - Message: "certificate object is nil in the proxy configuration", + Message: "Certificate object is nil in the proxy configuration", + PossibleActions: []string{ + "Check the logs of the Consul agent configuring the local proxy and ensure XDS updates are being sent to the proxy", + }, } return []validate.Message{msg} } @@ -26,7 +29,10 @@ func (t *Troubleshoot) validateCerts(certs *envoy_admin_v3.Certificates) validat if len(certs.GetCertificates()) == 0 { msg := validate.Message{ Success: false, - Message: "no certificates found", + Message: "No certificates found", + PossibleActions: []string{ + "Check the logs of the Consul agent configuring the local proxy and ensure XDS updates are being sent to the proxy", + }, } return []validate.Message{msg} } @@ -36,7 +42,10 @@ func (t *Troubleshoot) validateCerts(certs *envoy_admin_v3.Certificates) validat if now.After(cacert.GetExpirationTime().AsTime()) { msg := validate.Message{ Success: false, - Message: "ca certificate is expired", + Message: "CA certificate is expired", + PossibleActions: []string{ + "Check the logs of the Consul agent configuring the local proxy and ensure XDS updates are being sent to the proxy", + }, } certMessages = append(certMessages, msg) } @@ -46,7 +55,10 @@ func (t *Troubleshoot) validateCerts(certs *envoy_admin_v3.Certificates) validat if now.After(cc.GetExpirationTime().AsTime()) { msg := validate.Message{ Success: false, - Message: "certificate chain is expired", + Message: "Certificate chain is expired", + PossibleActions: []string{ + "Check the logs of the Consul agent configuring the local proxy and ensure XDS updates are being sent to the proxy", + }, } certMessages = append(certMessages, msg) } diff --git a/troubleshoot/proxy/certs_test.go b/troubleshoot/proxy/certs_test.go index 63f2a0256c18..fc03cb062ff2 100644 --- a/troubleshoot/proxy/certs_test.go +++ b/troubleshoot/proxy/certs_test.go @@ -21,13 +21,13 @@ func TestValidateCerts(t *testing.T) { }{ "cert is nil": { certs: nil, - expectedError: "certificate object is nil in the proxy configuration", + expectedError: "Certificate object is nil in the proxy configuration", }, "no certificates": { certs: &envoy_admin_v3.Certificates{ Certificates: []*envoy_admin_v3.Certificate{}, }, - expectedError: "no certificates found", + expectedError: "No certificates found", }, "ca expired": { certs: &envoy_admin_v3.Certificates{ @@ -41,7 +41,7 @@ func TestValidateCerts(t *testing.T) { }, }, }, - expectedError: "ca certificate is expired", + expectedError: "CA certificate is expired", }, "cert expired": { certs: &envoy_admin_v3.Certificates{ @@ -55,7 +55,7 @@ func TestValidateCerts(t *testing.T) { }, }, }, - expectedError: "certificate chain is expired", + expectedError: "Certificate chain is expired", }, } @@ -67,7 +67,9 @@ func TestValidateCerts(t *testing.T) { var outputErrors string for _, msgError := range messages.Errors() { outputErrors += msgError.Message - outputErrors += msgError.PossibleActions + for _, action := range msgError.PossibleActions { + outputErrors += action + } } if tc.expectedError == "" { require.True(t, messages.Success()) diff --git a/troubleshoot/proxy/stats.go b/troubleshoot/proxy/stats.go index 0fdb0e43ee70..a2ab88ffa6ba 100644 --- a/troubleshoot/proxy/stats.go +++ b/troubleshoot/proxy/stats.go @@ -3,6 +3,7 @@ package troubleshoot import ( "encoding/json" "fmt" + envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" "github.com/hashicorp/consul/troubleshoot/validate" ) @@ -32,10 +33,19 @@ func (t *Troubleshoot) troubleshootStats() (validate.Messages, error) { } } - statMessages = append(statMessages, validate.Message{ - Success: true, - Message: fmt.Sprintf("Envoy has %v rejected configurations", totalConfigRejections), - }) + if totalConfigRejections > 0 { + statMessages = append(statMessages, validate.Message{ + Message: fmt.Sprintf("Envoy has %v rejected configurations", totalConfigRejections), + PossibleActions: []string{ + "Check the logs of the Consul agent configuring the local proxy to see why Envoy rejected this configuration", + }, + }) + } else { + statMessages = append(statMessages, validate.Message{ + Success: true, + Message: fmt.Sprintf("Envoy has %v rejected configurations", totalConfigRejections), + }) + } connectionFailureStats, err := t.getEnvoyStats("upstream_cx_connect_fail") if err != nil { @@ -50,7 +60,7 @@ func (t *Troubleshoot) troubleshootStats() (validate.Messages, error) { } statMessages = append(statMessages, validate.Message{ Success: true, - Message: fmt.Sprintf("Envoy has detected %v connection failure(s).", totalCxFailures), + Message: fmt.Sprintf("Envoy has detected %v connection failure(s)", totalCxFailures), }) return statMessages, nil } diff --git a/troubleshoot/proxy/troubleshoot_proxy.go b/troubleshoot/proxy/troubleshoot_proxy.go index ff74aff0d4e7..8d1bfdc0bf17 100644 --- a/troubleshoot/proxy/troubleshoot_proxy.go +++ b/troubleshoot/proxy/troubleshoot_proxy.go @@ -10,15 +10,6 @@ import ( "github.com/hashicorp/consul/troubleshoot/validate" ) -const ( - listeners string = "type.googleapis.com/envoy.admin.v3.ListenersConfigDump" - clusters string = "type.googleapis.com/envoy.admin.v3.ClustersConfigDump" - routes string = "type.googleapis.com/envoy.admin.v3.RoutesConfigDump" - endpoints string = "type.googleapis.com/envoy.admin.v3.EndpointsConfigDump" - bootstrap string = "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump" - httpConnManager string = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" -) - type Troubleshoot struct { client *api.Client envoyAddr net.IPAddr @@ -79,7 +70,7 @@ func (t *Troubleshoot) RunAllTests(upstreamEnvoyID, upstreamIP string) (validate if errors := messages.Errors(); len(errors) == 0 { msg := validate.Message{ Success: true, - Message: "certificates are valid", + Message: "Certificates are valid", } allTestMessages = append(allTestMessages, msg) } @@ -97,7 +88,7 @@ func (t *Troubleshoot) RunAllTests(upstreamEnvoyID, upstreamIP string) (validate if errors := messages.Errors(); len(errors) == 0 { msg := validate.Message{ Success: true, - Message: "upstream resources are valid", + Message: "Upstream resources are valid", } allTestMessages = append(allTestMessages, msg) } diff --git a/troubleshoot/proxy/upstreams.go b/troubleshoot/proxy/upstreams.go index abd9711abfa2..2ac7c8e953da 100644 --- a/troubleshoot/proxy/upstreams.go +++ b/troubleshoot/proxy/upstreams.go @@ -2,6 +2,7 @@ package troubleshoot import ( "fmt" + envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" @@ -29,7 +30,7 @@ func (t *Troubleshoot) GetUpstreams() ([]string, []UpstreamIP, error) { for _, cfg := range t.envoyConfigDump.Configs { switch cfg.TypeUrl { - case listeners: + case listenersType: lcd := &envoy_admin_v3.ListenersConfigDump{} err := proto.Unmarshal(cfg.GetValue(), lcd) @@ -135,7 +136,7 @@ func getClustersFromRoutes(routeName string, cfgDump *envoy_admin_v3.ConfigDump) for _, cfg := range cfgDump.Configs { switch cfg.TypeUrl { - case routes: + case routesType: rcd := &envoy_admin_v3.RoutesConfigDump{} err := proto.Unmarshal(cfg.GetValue(), rcd) diff --git a/troubleshoot/proxy/upstreams_test.go b/troubleshoot/proxy/upstreams_test.go index 6493b5349d26..e1d6ff90753a 100644 --- a/troubleshoot/proxy/upstreams_test.go +++ b/troubleshoot/proxy/upstreams_test.go @@ -1,14 +1,15 @@ package troubleshoot import ( + "io" + "os" + "testing" + envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "io" - "os" - "testing" ) func TestGetUpstreamIPsFromFilterChain(t *testing.T) { @@ -61,7 +62,7 @@ func TestGetUpstreamIPsFromFilterChain(t *testing.T) { for _, cfg := range cfgDump.Configs { switch cfg.TypeUrl { - case listeners: + case listenersType: lcd := &envoy_admin_v3.ListenersConfigDump{} err := proto.Unmarshal(cfg.GetValue(), lcd) diff --git a/troubleshoot/validate/validate.go b/troubleshoot/validate/validate.go index 59dad4031a3b..01f950ba6db5 100644 --- a/troubleshoot/validate/validate.go +++ b/troubleshoot/validate/validate.go @@ -110,7 +110,7 @@ type Messages []Message type Message struct { Success bool Message string - PossibleActions string + PossibleActions []string } func (m Messages) Success() bool { @@ -137,6 +137,18 @@ func (m Messages) Errors() Messages { // GetMessages returns the error based only on Validate's state. func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator EndpointValidator, clusters *envoy_admin_v3.Clusters) Messages { var messages Messages + missingXDSActions := []string{ + "Check that your upstream service is registered with Consul", + "Make sure your upstream exists by running the `consul[-k8s] troubleshoot upstreams` command", + "If you are using transparent proxy for this upstream, ensure you have set up allow intentions to the upstream", + "Check the logs of the Consul agent configuring the local proxy to ensure XDS resources were sent by Consul", + } + missingEndpointsActions := []string{ + "Check that your upstream service is healthy and running", + "Check that your upstream service is registered with Consul", + "Check that the upstream proxy is healthy and running", + "If you are explicitly configuring upstreams, ensure the name of the upstream is correct", + } var upstream string upstream = v.envoyID @@ -145,19 +157,25 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin } if !v.listener { - messages = append(messages, Message{Message: fmt.Sprintf("no listener for upstream %q", upstream)}) + messages = append(messages, Message{ + Message: fmt.Sprintf("No listener for upstream %q", upstream), + PossibleActions: missingXDSActions, + }) } else { messages = append(messages, Message{ - Message: fmt.Sprintf("listener for upstream %q found", upstream), + Message: fmt.Sprintf("Listener for upstream %q found", upstream), Success: true, }) } if v.usesRDS && !v.route { - messages = append(messages, Message{Message: fmt.Sprintf("no route for upstream %q", upstream)}) - } else { messages = append(messages, Message{ - Message: fmt.Sprintf("route for upstream %q found", upstream), + Message: fmt.Sprintf("No route for upstream %q", upstream), + PossibleActions: missingXDSActions, + }) + } else if v.route { + messages = append(messages, Message{ + Message: fmt.Sprintf("Route for upstream %q found", upstream), Success: true, }) } @@ -173,11 +191,14 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin _, ok := v.snis[sni] if !ok || !resource.cluster { - messages = append(messages, Message{Message: fmt.Sprintf("no cluster %q for upstream %q", sni, upstream)}) + messages = append(messages, Message{ + Message: fmt.Sprintf("No cluster %q for upstream %q", sni, upstream), + PossibleActions: missingXDSActions, + }) continue } else { messages = append(messages, Message{ - Message: fmt.Sprintf("cluster %q for upstream %q found", sni, upstream), + Message: fmt.Sprintf("Cluster %q for upstream %q found", sni, upstream), Success: true, }) } @@ -186,7 +207,7 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin // validation. if strings.Contains(sni, "passthrough~") { messages = append(messages, Message{ - Message: fmt.Sprintf("cluster %q is a passthrough cluster, skipping endpoint healthiness check", sni), + Message: fmt.Sprintf("Cluster %q is a passthrough cluster, skipping endpoint healthiness check", sni), Success: true, }) continue @@ -206,10 +227,13 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin } } if !oneClusterHasEndpoints { - messages = append(messages, Message{Message: fmt.Sprintf("no healthy endpoints for aggregate cluster %q for upstream %q", sni, upstream)}) + messages = append(messages, Message{ + Message: fmt.Sprintf("No healthy endpoints for aggregate cluster %q for upstream %q", sni, upstream), + PossibleActions: missingEndpointsActions, + }) } else { messages = append(messages, Message{ - Message: fmt.Sprintf("healthy endpoints for aggregate cluster %q for upstream %q", sni, upstream), + Message: fmt.Sprintf("Healthy endpoints for aggregate cluster %q for upstream %q found", sni, upstream), Success: true, }) } @@ -218,11 +242,12 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin endpointValidator(resource, sni, clusters) if (resource.usesEDS && !resource.loadAssignment) || resource.endpoints == 0 { messages = append(messages, Message{ - Message: fmt.Sprintf("no healthy endpoints for cluster %q for upstream %q", sni, upstream), + Message: fmt.Sprintf("No healthy endpoints for cluster %q for upstream %q", sni, upstream), + PossibleActions: missingEndpointsActions, }) } else { messages = append(messages, Message{ - Message: fmt.Sprintf("healthy endpoints for cluster %q for upstream %q", sni, upstream), + Message: fmt.Sprintf("Healthy endpoints for cluster %q for upstream %q found", sni, upstream), Success: true, }) } @@ -235,7 +260,7 @@ func (v *Validate) GetMessages(validateEndpoints bool, endpointValidator Endpoin } if numRequiredResources == 0 { - messages = append(messages, Message{Message: fmt.Sprintf("no clusters found on route or listener")}) + messages = append(messages, Message{Message: fmt.Sprintf("No clusters found on route or listener")}) } return messages diff --git a/troubleshoot/validate/validate_test.go b/troubleshoot/validate/validate_test.go index c8f353f306ab..6496001b79ad 100644 --- a/troubleshoot/validate/validate_test.go +++ b/troubleshoot/validate/validate_test.go @@ -63,7 +63,7 @@ func TestErrors(t *testing.T) { r.loadAssignment = true r.endpoints = 1 }, - err: "no clusters found on route or listener", + err: "No clusters found on route or listener", }, "no healthy endpoints": { validate: func() *Validate { @@ -86,7 +86,7 @@ func TestErrors(t *testing.T) { endpointValidator: func(r *resource, s string, clusters *envoy_admin_v3.Clusters) { r.loadAssignment = true }, - err: "no healthy endpoints for cluster \"db-sni\" for upstream \"db\"", + err: "No healthy endpoints for cluster \"db-sni\" for upstream \"db\"", }, "success: aggregate cluster with one target with endpoints": { validate: func() *Validate { @@ -169,7 +169,7 @@ func TestErrors(t *testing.T) { r.loadAssignment = true r.endpoints = 0 }, - err: "no healthy endpoints for aggregate cluster \"db-sni\" for upstream \"db\"", + err: "No healthy endpoints for aggregate cluster \"db-sni\" for upstream \"db\"", }, "success: passthrough cluster doesn't error even though there are zero endpoints": { validate: func() *Validate { @@ -203,7 +203,9 @@ func TestErrors(t *testing.T) { var outputErrors string for _, msgError := range messages.Errors() { outputErrors += msgError.Message - outputErrors += msgError.PossibleActions + for _, action := range msgError.PossibleActions { + outputErrors += action + } } if tc.err == "" { require.True(t, messages.Success()) From 3cba165d783ce70630dacd2f76db2d90a8822ac9 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Feb 2023 14:24:01 -0500 Subject: [PATCH 007/421] Backport of Fix panicky xDS test flakes into release/1.15.x (#16310) * backport of commit 01ca6e268bd43597d3876f6736a954433e4eb644 * backport of commit 485cb4d88d25f064f418b9dbbbe2dd291a9d38a3 --------- Co-authored-by: Andrew Stucki --- agent/xds/testing.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/agent/xds/testing.go b/agent/xds/testing.go index 967a96e6acd7..76cc53f49fd2 100644 --- a/agent/xds/testing.go +++ b/agent/xds/testing.go @@ -85,6 +85,8 @@ type TestEnvoy struct { EnvoyVersion string deltaStream *TestADSDeltaStream // Incremental v3 + + closed bool } // NewTestEnvoy creates a TestEnvoy instance. @@ -225,9 +227,9 @@ func (e *TestEnvoy) Close() error { defer e.mu.Unlock() // unblock the recv chans to simulate recv errors when client disconnects - if e.deltaStream != nil && e.deltaStream.recvCh != nil { + if !e.closed && e.deltaStream.recvCh != nil { close(e.deltaStream.recvCh) - e.deltaStream = nil + e.closed = true } if e.cancel != nil { e.cancel() From 4920cb4a77ebb094d976b0d3ba74dbca764d5d31 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Feb 2023 14:46:49 -0500 Subject: [PATCH 008/421] Backport of Inline API Gateway TLS cert code into release/1.15.x (#16306) * Inline API Gateway TLS cert code (#16295) * Include secret type when building resources from config snapshot * First pass at generating envoy secrets from api-gateway snapshot * Update comments for xDS update order * Add secret type + corresponding golden files to existing tests * Initialize test helpers for testing api-gateway resource generation * Generate golden files for new api-gateway xDS resource test * Support ADS for TLS certificates on api-gateway * Configure TLS on api-gateway listeners * Inline TLS cert code * update tests * Add SNI support so we can have multiple certificates * Remove commented out section from helper * regen deep-copy * Add tcp tls test --------- Co-authored-by: Nathan Coleman * Fix bad merge --------- Co-authored-by: Andrew Stucki Co-authored-by: Nathan Coleman --- agent/proxycfg/proxycfg.deepcopy.go | 17 ++ agent/proxycfg/snapshot.go | 40 ++- agent/proxycfg/testing_api_gateway.go | 157 ++++++++++ .../config_entry_inline_certificate.go | 24 ++ agent/structs/config_entry_status.go | 5 + agent/xds/delta.go | 15 +- agent/xds/listeners.go | 3 + agent/xds/listeners_ingress.go | 168 +++++++++- agent/xds/listeners_test.go | 204 +++++++++++++ agent/xds/resources.go | 3 +- agent/xds/resources_test.go | 106 ++++++- agent/xds/secrets.go | 12 +- agent/xds/testcommon/testcommon.go | 3 + ...route-and-inline-certificate.latest.golden | 55 ++++ ...route-and-inline-certificate.latest.golden | 5 + ...ttp-listener-with-http-route.latest.golden | 49 +++ .../api-gateway-http-listener.latest.golden | 5 + ...api-gateway-nil-config-entry.latest.golden | 5 + ...ener-with-tcp-and-http-route.latest.golden | 74 +++++ ...-tcp-listener-with-tcp-route.latest.golden | 32 ++ .../api-gateway-tcp-listener.latest.golden | 5 + ...route-and-inline-certificate.latest.golden | 60 ++++ .../listeners/api-gateway.latest.golden | 5 + ...route-and-inline-certificate.latest.golden | 5 + ...route-and-inline-certificate.latest.golden | 5 + ...nect-proxy-exported-to-peers.latest.golden | 5 + ...and-failover-to-cluster-peer.latest.golden | 5 + ...and-redirect-to-cluster-peer.latest.golden | 5 + ...-proxy-with-peered-upstreams.latest.golden | 5 + .../testdata/secrets/defaults.latest.golden | 5 + ...ateway-with-peered-upstreams.latest.golden | 5 + ...ateway-peering-control-plane.latest.golden | 5 + ...ed-services-http-with-router.latest.golden | 5 + ...xported-peered-services-http.latest.golden | 5 + ...ith-exported-peered-services.latest.golden | 5 + ...ith-imported-peered-services.latest.golden | 5 + ...through-mesh-gateway-enabled.latest.golden | 5 + ...arent-proxy-destination-http.latest.golden | 5 + ...ransparent-proxy-destination.latest.golden | 5 + ...ng-gateway-destinations-only.latest.golden | 5 + ...-proxy-with-peered-upstreams.latest.golden | 5 + .../secrets/transparent-proxy.latest.golden | 5 + .../capture.sh | 3 + .../service_gateway.hcl | 4 + .../setup.sh | 287 ++++++++++++++++++ .../vars.sh | 3 + .../verify.bats | 48 +++ .../capture.sh | 3 + .../service_gateway.hcl | 4 + .../setup.sh | 279 +++++++++++++++++ .../vars.sh | 3 + .../verify.bats | 43 +++ test/integration/connect/envoy/helpers.bash | 14 + 53 files changed, 1807 insertions(+), 31 deletions(-) create mode 100644 agent/proxycfg/testing_api_gateway.go create mode 100644 agent/xds/testdata/clusters/api-gateway-with-tcp-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/endpoints/api-gateway-with-tcp-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-http-listener-with-http-route.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-http-listener.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-nil-config-entry.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-and-http-route.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-route.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-tcp-listener.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-with-tcp-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway.latest.golden create mode 100644 agent/xds/testdata/routes/api-gateway-with-tcp-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/secrets/api-gateway-with-tcp-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/secrets/connect-proxy-exported-to-peers.latest.golden create mode 100644 agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/secrets/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden create mode 100644 agent/xds/testdata/secrets/connect-proxy-with-peered-upstreams.latest.golden create mode 100644 agent/xds/testdata/secrets/defaults.latest.golden create mode 100644 agent/xds/testdata/secrets/local-mesh-gateway-with-peered-upstreams.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-peering-control-plane.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http-with-router.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-with-imported-peered-services.latest.golden create mode 100644 agent/xds/testdata/secrets/mesh-gateway-with-peer-through-mesh-gateway-enabled.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-destination-http.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-destination.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-terminating-gateway-destinations-only.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-with-peered-upstreams.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy.latest.golden create mode 100644 test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/capture.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/setup.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/vars.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/verify.bats create mode 100644 test/integration/connect/envoy/case-api-gateway-tcp-tls-overlapping-hosts/capture.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-tcp-tls-overlapping-hosts/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-api-gateway-tcp-tls-overlapping-hosts/setup.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-tcp-tls-overlapping-hosts/vars.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-tcp-tls-overlapping-hosts/verify.bats diff --git a/agent/proxycfg/proxycfg.deepcopy.go b/agent/proxycfg/proxycfg.deepcopy.go index 4c0318f67bfb..f1772ae72ed7 100644 --- a/agent/proxycfg/proxycfg.deepcopy.go +++ b/agent/proxycfg/proxycfg.deepcopy.go @@ -310,6 +310,23 @@ func (o *configSnapshotAPIGateway) DeepCopy() *configSnapshotAPIGateway { cp.Listeners[k2] = cp_Listeners_v2 } } + if o.ListenerCertificates != nil { + cp.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry, len(o.ListenerCertificates)) + for k2, v2 := range o.ListenerCertificates { + var cp_ListenerCertificates_v2 []structs.InlineCertificateConfigEntry + if v2 != nil { + cp_ListenerCertificates_v2 = make([]structs.InlineCertificateConfigEntry, len(v2)) + copy(cp_ListenerCertificates_v2, v2) + for i3 := range v2 { + { + retV := v2[i3].DeepCopy() + cp_ListenerCertificates_v2[i3] = *retV + } + } + } + cp.ListenerCertificates[k2] = cp_ListenerCertificates_v2 + } + } if o.BoundListeners != nil { cp.BoundListeners = make(map[string]structs.BoundAPIGatewayListener, len(o.BoundListeners)) for k2, v2 := range o.BoundListeners { diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 13c854005621..6ce5a30083eb 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -734,6 +734,8 @@ type configSnapshotAPIGateway struct { // Listeners is the original listener config from the api-gateway config // entry to save us trying to pass fields through Upstreams Listeners map[string]structs.APIGatewayListener + // this acts as an intermediary for inlining certificates + ListenerCertificates map[IngressListenerKey][]structs.InlineCertificateConfigEntry BoundListeners map[string]structs.BoundAPIGatewayListener } @@ -751,6 +753,9 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI watchedUpstreamEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes) watchedGatewayEndpoints := make(map[UpstreamID]map[string]structs.CheckServiceNodes) + // reset the cached certificates + c.ListenerCertificates = make(map[IngressListenerKey][]structs.InlineCertificateConfigEntry) + for name, listener := range c.Listeners { boundListener, ok := c.BoundListeners[name] if !ok { @@ -802,17 +807,18 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI watchedGatewayEndpoints[id] = gatewayEndpoints } + key := IngressListenerKey{ + Port: listener.Port, + Protocol: string(listener.Protocol), + } + // Configure TLS for the ingress listener - tls, err := c.toIngressTLS() + tls, err := c.toIngressTLS(key, listener, boundListener) if err != nil { return configSnapshotIngressGateway{}, err } - ingressListener.TLS = tls - key := IngressListenerKey{ - Port: listener.Port, - Protocol: string(listener.Protocol), - } + ingressListener.TLS = tls ingressListeners[key] = ingressListener ingressUpstreams[key] = upstreams } @@ -905,9 +911,25 @@ DOMAIN_LOOP: return services, upstreams, compiled, err } -func (c *configSnapshotAPIGateway) toIngressTLS() (*structs.GatewayTLSConfig, error) { - // TODO (t-eckert) this is dependent on future SDS work. - return &structs.GatewayTLSConfig{}, nil +func (c *configSnapshotAPIGateway) toIngressTLS(key IngressListenerKey, listener structs.APIGatewayListener, bound structs.BoundAPIGatewayListener) (*structs.GatewayTLSConfig, error) { + if len(listener.TLS.Certificates) == 0 { + return nil, nil + } + + for _, certRef := range bound.Certificates { + cert, ok := c.Certificates.Get(certRef) + if !ok { + continue + } + c.ListenerCertificates[key] = append(c.ListenerCertificates[key], *cert) + } + + return &structs.GatewayTLSConfig{ + Enabled: true, + TLSMinVersion: listener.TLS.MinVersion, + TLSMaxVersion: listener.TLS.MaxVersion, + CipherSuites: listener.TLS.CipherSuites, + }, nil } type configSnapshotIngressGateway struct { diff --git a/agent/proxycfg/testing_api_gateway.go b/agent/proxycfg/testing_api_gateway.go new file mode 100644 index 000000000000..dd55f2eec5c2 --- /dev/null +++ b/agent/proxycfg/testing_api_gateway.go @@ -0,0 +1,157 @@ +package proxycfg + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/consul/discoverychain" + "github.com/mitchellh/go-testing-interface" + + "github.com/hashicorp/consul/agent/structs" +) + +func TestConfigSnapshotAPIGateway( + t testing.T, + variation string, + nsFn func(ns *structs.NodeService), + configFn func(entry *structs.APIGatewayConfigEntry, boundEntry *structs.BoundAPIGatewayConfigEntry), + routes []structs.BoundRoute, + certificates []structs.InlineCertificateConfigEntry, + extraUpdates []UpdateEvent, + additionalEntries ...structs.ConfigEntry, +) *ConfigSnapshot { + roots, placeholderLeaf := TestCerts(t) + + entry := &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "api-gateway", + } + boundEntry := &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "api-gateway", + } + + if configFn != nil { + configFn(entry, boundEntry) + } + + baseEvents := []UpdateEvent{ + { + CorrelationID: rootsWatchID, + Result: roots, + }, + { + CorrelationID: leafWatchID, + Result: placeholderLeaf, + }, + { + CorrelationID: gatewayConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: entry, + }, + }, + { + CorrelationID: gatewayConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: boundEntry, + }, + }, + } + + for _, route := range routes { + // Add the watch event for the route. + watch := UpdateEvent{ + CorrelationID: routeConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: route, + }, + } + baseEvents = append(baseEvents, watch) + + // Add the watch event for the discovery chain. + entries := []structs.ConfigEntry{ + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": route.GetProtocol(), + }, + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "api-gateway", + }, + } + + // Add a discovery chain watch event for each service. + for _, serviceName := range route.GetServiceNames() { + discoChain := UpdateEvent{ + CorrelationID: fmt.Sprintf("discovery-chain:%s", UpstreamIDString("", "", serviceName.Name, &serviceName.EnterpriseMeta, "")), + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, serviceName.Name, "default", "default", "dc1", connect.TestClusterID+".consul", nil, entries...), + }, + } + baseEvents = append(baseEvents, discoChain) + } + } + + for _, certificate := range certificates { + inlineCertificate := certificate + baseEvents = append(baseEvents, UpdateEvent{ + CorrelationID: inlineCertificateConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: &inlineCertificate, + }, + }) + } + + upstreams := structs.TestUpstreams(t) + + baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot( + t, variation, upstreams, additionalEntries..., + )) + + return testConfigSnapshotFixture(t, &structs.NodeService{ + Kind: structs.ServiceKindAPIGateway, + Service: "api-gateway", + Address: "1.2.3.4", + Meta: nil, + TaggedAddresses: nil, + }, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates)) +} + +// TestConfigSnapshotAPIGateway_NilConfigEntry is used to test when +// the update event for the config entry returns nil +// since this always happens on the first watch if it doesn't exist. +func TestConfigSnapshotAPIGateway_NilConfigEntry( + t testing.T, +) *ConfigSnapshot { + roots, _ := TestCerts(t) + + baseEvents := []UpdateEvent{ + { + CorrelationID: rootsWatchID, + Result: roots, + }, + { + CorrelationID: gatewayConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist. + }, + }, + { + CorrelationID: gatewayConfigWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist. + }, + }, + } + + return testConfigSnapshotFixture(t, &structs.NodeService{ + Kind: structs.ServiceKindAPIGateway, + Service: "api-gateway", + Address: "1.2.3.4", + Meta: nil, + TaggedAddresses: nil, + }, nil, nil, testSpliceEvents(baseEvents, nil)) +} diff --git a/agent/structs/config_entry_inline_certificate.go b/agent/structs/config_entry_inline_certificate.go index 24028ffff2fa..18e3c7716db3 100644 --- a/agent/structs/config_entry_inline_certificate.go +++ b/agent/structs/config_entry_inline_certificate.go @@ -64,6 +64,30 @@ func (e *InlineCertificateConfigEntry) Validate() error { return nil } +func (e *InlineCertificateConfigEntry) Hosts() ([]string, error) { + certificateBlock, _ := pem.Decode([]byte(e.Certificate)) + if certificateBlock == nil { + return nil, errors.New("failed to parse certificate PEM") + } + + certificate, err := x509.ParseCertificate(certificateBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse certificate: %w", err) + } + + hosts := []string{certificate.Subject.CommonName} + + for _, name := range certificate.DNSNames { + hosts = append(hosts, name) + } + + for _, ip := range certificate.IPAddresses { + hosts = append(hosts, ip.String()) + } + + return hosts, nil +} + func (e *InlineCertificateConfigEntry) CanRead(authz acl.Authorizer) error { var authzContext acl.AuthorizerContext e.FillAuthzContext(&authzContext) diff --git a/agent/structs/config_entry_status.go b/agent/structs/config_entry_status.go index faafd34d7b7a..5485a6ccaf95 100644 --- a/agent/structs/config_entry_status.go +++ b/agent/structs/config_entry_status.go @@ -1,6 +1,7 @@ package structs import ( + "fmt" "sort" "time" @@ -24,6 +25,10 @@ type ResourceReference struct { acl.EnterpriseMeta } +func (r *ResourceReference) String() string { + return fmt.Sprintf("%s:%s/%s/%s/%s", r.Kind, r.PartitionOrDefault(), r.NamespaceOrDefault(), r.Name, r.SectionName) +} + func (r *ResourceReference) IsSame(other *ResourceReference) bool { if r == nil && other == nil { return true diff --git a/agent/xds/delta.go b/agent/xds/delta.go index 5a1ef17d2445..62a725b8f1b1 100644 --- a/agent/xds/delta.go +++ b/agent/xds/delta.go @@ -466,20 +466,21 @@ func (s *Server) applyEnvoyExtensions(resources *xdscommon.IndexedResources, cfg return nil } +// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#eventual-consistency-considerations var xDSUpdateOrder = []xDSUpdateOperation{ - // TODO Update comments + // 1. SDS updates (if any) can be pushed here with no harm. {TypeUrl: xdscommon.SecretType, Upsert: true}, - // 1. CDS updates (if any) must always be pushed first. + // 2. CDS updates (if any) must always be pushed before the following types. {TypeUrl: xdscommon.ClusterType, Upsert: true}, - // 2. EDS updates (if any) must arrive after CDS updates for the respective clusters. + // 3. EDS updates (if any) must arrive after CDS updates for the respective clusters. {TypeUrl: xdscommon.EndpointType, Upsert: true}, - // 3. LDS updates must arrive after corresponding CDS/EDS updates. + // 4. LDS updates must arrive after corresponding CDS/EDS updates. {TypeUrl: xdscommon.ListenerType, Upsert: true, Remove: true}, - // 4. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates. + // 5. RDS updates related to the newly added listeners must arrive after CDS/EDS/LDS updates. {TypeUrl: xdscommon.RouteType, Upsert: true, Remove: true}, - // 5. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates. + // 6. (NOT IMPLEMENTED YET IN CONSUL) VHDS updates (if any) related to the newly added RouteConfigurations must arrive after RDS updates. // {}, - // 6. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed. + // 7. Stale CDS clusters, related EDS endpoints (ones no longer being referenced) and SDS secrets can then be removed. {TypeUrl: xdscommon.ClusterType, Remove: true}, {TypeUrl: xdscommon.EndpointType, Remove: true}, {TypeUrl: xdscommon.SecretType, Remove: true}, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 291e71517641..0c31c423849f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -2328,6 +2328,9 @@ func makeHTTPInspectorListenerFilter() (*envoy_listener_v3.ListenerFilter, error } func makeSNIFilterChainMatch(sniMatches ...string) *envoy_listener_v3.FilterChainMatch { + if sniMatches == nil { + return nil + } return &envoy_listener_v3.FilterChainMatch{ ServerNames: sniMatches, } diff --git a/agent/xds/listeners_ingress.go b/agent/xds/listeners_ingress.go index 6083529849dc..40eb24230e0f 100644 --- a/agent/xds/listeners_ingress.go +++ b/agent/xds/listeners_ingress.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/types" ) @@ -25,6 +26,15 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap return nil, fmt.Errorf("no listener config found for listener on proto/port %s/%d", listenerKey.Protocol, listenerKey.Port) } + var isAPIGatewayWithTLS bool + var certs []structs.InlineCertificateConfigEntry + if cfgSnap.APIGateway.ListenerCertificates != nil { + certs = cfgSnap.APIGateway.ListenerCertificates[listenerKey] + } + if certs != nil { + isAPIGatewayWithTLS = true + } + tlsContext, err := makeDownstreamTLSContextFromSnapshotListenerConfig(cfgSnap, listenerCfg) if err != nil { return nil, err @@ -72,6 +82,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap logger: s.Logger, } l := makeListener(opts) + filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ accessLogs: &cfgSnap.Proxy.AccessLogs, routeName: uid.EnvoyID(), @@ -87,8 +98,30 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap l.FilterChains = []*envoy_listener_v3.FilterChain{ filterChain, } - resources = append(resources, l) + if isAPIGatewayWithTLS { + // construct SNI filter chains + l.FilterChains, err = makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey, listenerFilterOpts{ + useRDS: useRDS, + protocol: listenerKey.Protocol, + routeName: listenerKey.RouteName(), + cluster: clusterName, + statPrefix: "ingress_upstream_", + accessLogs: &cfgSnap.Proxy.AccessLogs, + logger: s.Logger, + }, certs) + if err != nil { + return nil, err + } + // add the tls inspector to do SNI introspection + tlsInspector, err := makeTLSInspectorListenerFilter() + if err != nil { + return nil, err + } + l.ListenerFilters = []*envoy_listener_v3.ListenerFilter{tlsInspector} + } + + resources = append(resources, l) } else { // If multiple upstreams share this port, make a special listener for the protocol. listenerOpts := makeListenerOpts{ @@ -121,6 +154,13 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap return nil, err } + if isAPIGatewayWithTLS { + sniFilterChains, err = makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey, filterOpts, certs) + if err != nil { + return nil, err + } + } + // If there are any sni filter chains, we need a TLS inspector filter! if len(sniFilterChains) > 0 { tlsInspector, err := makeTLSInspectorListenerFilter() @@ -134,7 +174,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap // See if there are other services that didn't have specific SNI-matching // filter chains. If so add a default filterchain to serve them. - if len(sniFilterChains) < len(upstreams) { + if len(sniFilterChains) < len(upstreams) && !isAPIGatewayWithTLS { defaultFilter, err := makeListenerFilter(filterOpts) if err != nil { return nil, err @@ -379,10 +419,133 @@ func makeSDSOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot, return chains, nil } +// when we have multiple certificates on a single listener, we need +// to duplicate the filter chains with multiple TLS contexts +func makeInlineOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot, + tlsCfg structs.GatewayTLSConfig, + listenerKey proxycfg.IngressListenerKey, + filterOpts listenerFilterOpts, + certs []structs.InlineCertificateConfigEntry) ([]*envoy_listener_v3.FilterChain, error) { + + listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey] + if !ok { + return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port) + } + + var chains []*envoy_listener_v3.FilterChain + + constructChain := func(name string, hosts []string, tlsContext *envoy_tls_v3.CommonTlsContext) error { + filterOpts.filterName = name + filter, err := makeListenerFilter(filterOpts) + if err != nil { + return err + } + + // Configure alpn protocols on TLSContext + tlsContext.AlpnProtocols = getAlpnProtocols(listenerCfg.Protocol) + transportSocket, err := makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{ + CommonTlsContext: tlsContext, + RequireClientCertificate: &wrapperspb.BoolValue{Value: false}, + }) + if err != nil { + return err + } + + chains = append(chains, &envoy_listener_v3.FilterChain{ + FilterChainMatch: makeSNIFilterChainMatch(hosts...), + Filters: []*envoy_listener_v3.Filter{ + filter, + }, + TransportSocket: transportSocket, + }) + + return nil + } + + multipleCerts := len(certs) > 1 + + allCertHosts := map[string]struct{}{} + overlappingHosts := map[string]struct{}{} + + if multipleCerts { + // we only need to prune out overlapping hosts if we have more than + // one certificate + for _, cert := range certs { + hosts, err := cert.Hosts() + if err != nil { + return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts) + } + for _, host := range hosts { + if _, ok := allCertHosts[host]; ok { + overlappingHosts[host] = struct{}{} + } + allCertHosts[host] = struct{}{} + } + } + } + + for _, cert := range certs { + var hosts []string + + // if we only have one cert, we just use it for all ingress + if multipleCerts { + // otherwise, we need an SNI per cert and to fallback to our ingress + // gateway certificate signed by our Consul CA + certHosts, err := cert.Hosts() + if err != nil { + return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts) + } + // filter out any overlapping hosts so we don't have collisions in our filter chains + for _, host := range certHosts { + if _, ok := overlappingHosts[host]; !ok { + hosts = append(hosts, host) + } + } + + if len(hosts) == 0 { + // all of our hosts are overlapping, so we just skip this filter and it'll be + // handled by the default filter chain + continue + } + } + + if err := constructChain(cert.Name, hosts, makeInlineTLSContextFromGatewayTLSConfig(tlsCfg, cert)); err != nil { + return nil, err + } + } + + if multipleCerts { + // if we have more than one cert, add a default handler that uses the leaf cert from connect + if err := constructChain("default", nil, makeCommonTLSContext(cfgSnap.Leaf(), cfgSnap.RootPEMs(), makeTLSParametersFromGatewayTLSConfig(tlsCfg))); err != nil { + return nil, err + } + } + + return chains, nil +} + func makeTLSParametersFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *envoy_tls_v3.TlsParameters { return makeTLSParametersFromTLSConfig(tlsCfg.TLSMinVersion, tlsCfg.TLSMaxVersion, tlsCfg.CipherSuites) } +func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, cert structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext { + return &envoy_tls_v3.CommonTlsContext{ + TlsParams: makeTLSParametersFromGatewayTLSConfig(tlsCfg), + TlsCertificates: []*envoy_tls_v3.TlsCertificate{{ + CertificateChain: &envoy_core_v3.DataSource{ + Specifier: &envoy_core_v3.DataSource_InlineString{ + InlineString: lib.EnsureTrailingNewline(cert.Certificate), + }, + }, + PrivateKey: &envoy_core_v3.DataSource{ + Specifier: &envoy_core_v3.DataSource_InlineString{ + InlineString: lib.EnsureTrailingNewline(cert.PrivateKey), + }, + }, + }}, + } +} + func makeCommonTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *envoy_tls_v3.CommonTlsContext { return &envoy_tls_v3.CommonTlsContext{ TlsParams: makeTLSParametersFromGatewayTLSConfig(tlsCfg), @@ -396,6 +559,7 @@ func makeCommonTLSContextFromGatewayServiceTLSConfig(tlsCfg structs.GatewayServi TlsCertificateSdsSecretConfigs: makeTLSCertificateSdsSecretConfigsFromSDS(*tlsCfg.SDS), } } + func makeTLSCertificateSdsSecretConfigsFromSDS(sdsCfg structs.GatewayTLSSDSConfig) []*envoy_tls_v3.SdsSecretConfig { return []*envoy_tls_v3.SdsSecretConfig{ { diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index fffe3dc342df..6d0d64407bbf 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -527,6 +527,210 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, }, + { + name: "api-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, nil, nil, nil, nil) + }, + }, + { + name: "api-gateway-nil-config-entry", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway_NilConfigEntry(t) + }, + }, + { + name: "api-gateway-tcp-listener", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolTCP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + }, + } + }, nil, nil, nil) + }, + }, + { + name: "api-gateway-tcp-listener-with-tcp-route", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolTCP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Routes: []structs.ResourceReference{ + { + Name: "tcp-route", + Kind: structs.TCPRoute, + }, + }, + }, + } + + }, []structs.BoundRoute{ + &structs.TCPRouteConfigEntry{ + Name: "tcp-route", + Kind: structs.TCPRoute, + Parents: []structs.ResourceReference{ + { + Kind: structs.APIGateway, + Name: "api-gateway", + }, + }, + Services: []structs.TCPService{ + {Name: "tcp-service"}, + }, + }, + }, nil, nil) + }, + }, + { + name: "api-gateway-http-listener", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolHTTP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Routes: []structs.ResourceReference{}, + }, + } + }, nil, nil, nil) + }, + }, + { + name: "api-gateway-http-listener-with-http-route", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolHTTP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Routes: []structs.ResourceReference{ + { + Name: "http-route", + Kind: structs.HTTPRoute, + }, + }, + }, + } + }, []structs.BoundRoute{ + &structs.HTTPRouteConfigEntry{ + Name: "http-route", + Kind: structs.HTTPRoute, + Parents: []structs.ResourceReference{ + { + Kind: structs.APIGateway, + Name: "api-gateway", + }, + }, + Rules: []structs.HTTPRouteRule{ + { + Services: []structs.HTTPService{ + {Name: "http-service"}, + }, + }, + }, + }, + }, nil, nil) + }, + }, + { + name: "api-gateway-tcp-listener-with-tcp-and-http-route", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener-tcp", + Protocol: structs.ListenerProtocolTCP, + Port: 8080, + }, + { + Name: "listener-http", + Protocol: structs.ListenerProtocolHTTP, + Port: 8081, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener-tcp", + Routes: []structs.ResourceReference{ + { + Name: "tcp-route", + Kind: structs.TCPRoute, + }, + }, + }, + { + Name: "listener-http", + Routes: []structs.ResourceReference{ + { + Name: "http-route", + Kind: structs.HTTPRoute, + }, + }, + }, + } + }, []structs.BoundRoute{ + &structs.TCPRouteConfigEntry{ + Name: "tcp-route", + Kind: structs.TCPRoute, + Parents: []structs.ResourceReference{ + { + Kind: structs.APIGateway, + Name: "api-gateway", + }, + }, + Services: []structs.TCPService{ + {Name: "tcp-service"}, + }, + }, + &structs.HTTPRouteConfigEntry{ + Name: "http-route", + Kind: structs.HTTPRoute, + Parents: []structs.ResourceReference{ + { + Kind: structs.APIGateway, + Name: "api-gateway", + }, + }, + Rules: []structs.HTTPRouteRule{ + { + Services: []structs.HTTPService{ + {Name: "http-service"}, + }, + }, + }, + }, + }, nil, nil) + }, + }, { name: "ingress-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/resources.go b/agent/xds/resources.go index 6bd6709343c1..cab494da4cad 100644 --- a/agent/xds/resources.go +++ b/agent/xds/resources.go @@ -35,8 +35,7 @@ func NewResourceGenerator( func (g *ResourceGenerator) AllResourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) (map[string][]proto.Message, error) { all := make(map[string][]proto.Message) - // TODO Add xdscommon.SecretType - for _, typeUrl := range []string{xdscommon.ListenerType, xdscommon.RouteType, xdscommon.ClusterType, xdscommon.EndpointType} { + for _, typeUrl := range []string{xdscommon.ListenerType, xdscommon.RouteType, xdscommon.ClusterType, xdscommon.EndpointType, xdscommon.SecretType} { res, err := g.resourcesFromSnapshot(typeUrl, cfgSnap) if err != nil { return nil, fmt.Errorf("failed to generate xDS resources for %q: %v", typeUrl, err) diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index ad3c1b396c2b..5fc31dc9e2d9 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -9,6 +9,8 @@ import ( envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -25,6 +27,7 @@ var testTypeUrlToPrettyName = map[string]string{ xdscommon.RouteType: "routes", xdscommon.ClusterType: "clusters", xdscommon.EndpointType: "endpoints", + xdscommon.SecretType: "secrets", } type goldenTestCase struct { @@ -60,7 +63,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { // We need to replace the TLS certs with deterministic ones to make golden // files workable. Note we don't update these otherwise they'd change - // golder files for every test case and so not be any use! + // golden files for every test case and so not be any use! testcommon.SetupTLSRootsAndLeaf(t, snap) if tt.setup != nil { @@ -82,6 +85,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { xdscommon.RouteType, xdscommon.ClusterType, xdscommon.EndpointType, + xdscommon.SecretType, } require.Len(t, resources, len(typeUrls)) @@ -101,6 +105,8 @@ func TestAllResourcesFromSnapshot(t *testing.T) { return items[i].(*envoy_cluster_v3.Cluster).Name < items[j].(*envoy_cluster_v3.Cluster).Name case xdscommon.EndpointType: return items[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < items[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName + case xdscommon.SecretType: + return items[i].(*envoy_tls_v3.Secret).Name < items[j].(*envoy_tls_v3.Secret).Name default: panic("not possible") } @@ -165,6 +171,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) tests = append(tests, getTrafficControlPeeringGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...) + tests = append(tests, getAPIGatewayGoldenTestCases()...) latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { @@ -256,3 +263,100 @@ func getTrafficControlPeeringGoldenTestCases() []goldenTestCase { }, } } + +const ( + gatewayTestPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAx95Opa6t4lGEpiTUogEBptqOdam2ch4BHQGhNhX/MrDwwuZQ +httBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2jQlhqTodElkbsd5vWY8R/bxJWQSo +NvVE12TlzECxGpJEiHt4W0r8pGffk+rvpljiUyCfnT1kGF3znOSjK1hRMTn6RKWC +yYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409g9X5VU88/Bmmrz4cMyxce86Kc2ug +5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftrXOvuCbO5IBRHMOBHiHTZ4rtGuhMa +Ir21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+WmQIDAQABAoIBACYvceUzp2MK4gYA +GWPOP2uKbBdM0l+hHeNV0WAM+dHMfmMuL4pkT36ucqt0ySOLjw6rQyOZG5nmA6t9 +sv0g4ae2eCMlyDIeNi1Yavu4Wt6YX4cTXbQKThm83C6W2X9THKbauBbxD621bsDK +7PhiGPN60yPue7YwFQAPqqD4YaK+s22HFIzk9gwM/rkvAUNwRv7SyHMiFe4Igc1C +Eev7iHWzvj5Heoz6XfF+XNF9DU+TieSUAdjd56VyUb8XL4+uBTOhHwLiXvAmfaMR +HvpcxeKnYZusS6NaOxcUHiJnsLNWrxmJj9WEGgQzuLxcLjTe4vVmELVZD8t3QUKj +PAxu8tUCgYEA7KIWVn9dfVpokReorFym+J8FzLwSktP9RZYEMonJo00i8aii3K9s +u/aSwRWQSCzmON1ZcxZzWhwQF9usz6kGCk//9+4hlVW90GtNK0RD+j7sp4aT2JI8 +9eLEjTG+xSXa7XWe98QncjjL9lu/yrRncSTxHs13q/XP198nn2aYuQ8CgYEA2Dnt +sRBzv0fFEvzzFv7G/5f85mouN38TUYvxNRTjBLCXl9DeKjDkOVZ2b6qlfQnYXIru +H+W+v+AZEb6fySXc8FRab7lkgTMrwE+aeI4rkW7asVwtclv01QJ5wMnyT84AgDD/ +Dgt/RThFaHgtU9TW5GOZveL+l9fVPn7vKFdTJdcCgYEArJ99zjHxwJ1whNAOk1av +09UmRPm6TvRo4heTDk8oEoIWCNatoHI0z1YMLuENNSnT9Q280FFDayvnrY/qnD7A +kktT/sjwJOG8q8trKzIMqQS4XWm2dxoPcIyyOBJfCbEY6XuRsUuePxwh5qF942EB +yS9a2s6nC4Ix0lgPrqAIr48CgYBgS/Q6riwOXSU8nqCYdiEkBYlhCJrKpnJxF9T1 +ofa0yPzKZP/8ZEfP7VzTwHjxJehQ1qLUW9pG08P2biH1UEKEWdzo8vT6wVJT1F/k +HtTycR8+a+Hlk2SHVRHqNUYQGpuIe8mrdJ1as4Pd0d/F/P0zO9Rlh+mAsGPM8HUM +T0+9gwKBgHDoerX7NTskg0H0t8O+iSMevdxpEWp34ZYa9gHiftTQGyrRgERCa7Gj +nZPAxKb2JoWyfnu3v7G5gZ8fhDFsiOxLbZv6UZJBbUIh1MjJISpXrForDrC2QNLX +kHrHfwBFDB3KMudhQknsJzEJKCL/KmFH6o0MvsoaT9yzEl3K+ah/ +-----END RSA PRIVATE KEY-----` + gatewayTestCertificate = `-----BEGIN CERTIFICATE----- +MIICljCCAX4CCQCQMDsYO8FrPjANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV +UzAeFw0yMjEyMjAxNzUwMjVaFw0yNzEyMTkxNzUwMjVaMA0xCzAJBgNVBAYTAlVT +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx95Opa6t4lGEpiTUogEB +ptqOdam2ch4BHQGhNhX/MrDwwuZQhttBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2 +jQlhqTodElkbsd5vWY8R/bxJWQSoNvVE12TlzECxGpJEiHt4W0r8pGffk+rvplji +UyCfnT1kGF3znOSjK1hRMTn6RKWCyYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409 +g9X5VU88/Bmmrz4cMyxce86Kc2ug5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftr +XOvuCbO5IBRHMOBHiHTZ4rtGuhMaIr21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+W +mQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBfCqoUIdPf/HGSbOorPyZWbyizNtHJ +GL7x9cAeIYxpI5Y/WcO1o5v94lvrgm3FNfJoGKbV66+JxOge731FrfMpHplhar1Z +RahYIzNLRBTLrwadLAZkApUpZvB8qDK4knsTWFYujNsylCww2A6ajzIMFNU4GkUK +NtyHRuD+KYRmjXtyX1yHNqfGN3vOQmwavHq2R8wHYuBSc6LAHHV9vG+j0VsgMELO +qwxn8SmLkSKbf2+MsQVzLCXXN5u+D8Yv+4py+oKP4EQ5aFZuDEx+r/G/31rTthww +AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r +-----END CERTIFICATE-----` +) + +func getAPIGatewayGoldenTestCases() []goldenTestCase { + return []goldenTestCase{ + { + name: "api-gateway-with-tcp-route-and-inline-certificate", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolTCP, + Port: 8080, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "certificate", + }}, + }, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "certificate", + }}, + Routes: []structs.ResourceReference{{ + Kind: structs.TCPRoute, + Name: "route", + }}, + }, + } + }, []structs.BoundRoute{ + &structs.TCPRouteConfigEntry{ + Kind: structs.TCPRoute, + Name: "route", + Services: []structs.TCPService{{ + Name: "service", + }}, + }, + }, []structs.InlineCertificateConfigEntry{{ + Kind: structs.InlineCertificate, + Name: "certificate", + PrivateKey: gatewayTestPrivateKey, + Certificate: gatewayTestCertificate, + }}, nil) + }, + }, + } +} diff --git a/agent/xds/secrets.go b/agent/xds/secrets.go index 5547b6d2037d..a1ef50e44caa 100644 --- a/agent/xds/secrets.go +++ b/agent/xds/secrets.go @@ -21,18 +21,10 @@ func (s *ResourceGenerator) secretsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot case structs.ServiceKindConnectProxy, structs.ServiceKindTerminatingGateway, structs.ServiceKindMeshGateway, - structs.ServiceKindIngressGateway: + structs.ServiceKindIngressGateway, + structs.ServiceKindAPIGateway: return nil, nil - // Only API gateways utilize secrets - case structs.ServiceKindAPIGateway: - return s.secretsFromSnapshotAPIGateway(cfgSnap) default: return nil, fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) } } - -func (s *ResourceGenerator) secretsFromSnapshotAPIGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - var res []proto.Message - // TODO - return res, nil -} diff --git a/agent/xds/testcommon/testcommon.go b/agent/xds/testcommon/testcommon.go index c310d0980a62..ba75cf9ae978 100644 --- a/agent/xds/testcommon/testcommon.go +++ b/agent/xds/testcommon/testcommon.go @@ -22,6 +22,9 @@ func SetupTLSRootsAndLeaf(t *testing.T, snap *proxycfg.ConfigSnapshot) { case structs.ServiceKindMeshGateway: snap.MeshGateway.Leaf.CertPEM = loadTestResource(t, "test-leaf-cert") snap.MeshGateway.Leaf.PrivateKeyPEM = loadTestResource(t, "test-leaf-key") + case structs.ServiceKindAPIGateway: + snap.APIGateway.Leaf.CertPEM = loadTestResource(t, "test-leaf-cert") + snap.APIGateway.Leaf.PrivateKeyPEM = loadTestResource(t, "test-leaf-key") } } if snap.Roots != nil { diff --git a/agent/xds/testdata/clusters/api-gateway-with-tcp-route-and-inline-certificate.latest.golden b/agent/xds/testdata/clusters/api-gateway-with-tcp-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..e20479dfd1cf --- /dev/null +++ b/agent/xds/testdata/clusters/api-gateway-with-tcp-route-and-inline-certificate.latest.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/service" + } + ] + } + }, + "sni": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/api-gateway-with-tcp-route-and-inline-certificate.latest.golden b/agent/xds/testdata/endpoints/api-gateway-with-tcp-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..47b46bca225b --- /dev/null +++ b/agent/xds/testdata/endpoints/api-gateway-with-tcp-route-and-inline-certificate.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-http-listener-with-http-route.latest.golden b/agent/xds/testdata/listeners/api-gateway-http-listener-with-http-route.latest.golden new file mode 100644 index 000000000000..e9bee988de93 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-http-listener-with-http-route.latest.golden @@ -0,0 +1,49 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_8080", + "rds": { + "configSource": { + "ads": {}, + "resourceApiVersion": "V3" + }, + "routeConfigName": "8080" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": {} + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-http-listener.latest.golden b/agent/xds/testdata/listeners/api-gateway-http-listener.latest.golden new file mode 100644 index 000000000000..d2d839adf956 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-http-listener.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-nil-config-entry.latest.golden b/agent/xds/testdata/listeners/api-gateway-nil-config-entry.latest.golden new file mode 100644 index 000000000000..d2d839adf956 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-nil-config-entry.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-and-http-route.latest.golden b/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-and-http-route.latest.golden new file mode 100644 index 000000000000..9e136780767a --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-and-http-route.latest.golden @@ -0,0 +1,74 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:8081", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8081 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_8081", + "rds": { + "configSource": { + "ads": {}, + "resourceApiVersion": "V3" + }, + "routeConfigName": "8081" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": {} + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "tcp-service:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.tcp-service.default.default.dc1", + "cluster": "tcp-service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-route.latest.golden b/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-route.latest.golden new file mode 100644 index 000000000000..ffcb5830b9a4 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-tcp-listener-with-tcp-route.latest.golden @@ -0,0 +1,32 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "tcp-service:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.tcp-service.default.default.dc1", + "cluster": "tcp-service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-tcp-listener.latest.golden b/agent/xds/testdata/listeners/api-gateway-tcp-listener.latest.golden new file mode 100644 index 000000000000..d2d839adf956 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-tcp-listener.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-with-tcp-route-and-inline-certificate.latest.golden b/agent/xds/testdata/listeners/api-gateway-with-tcp-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..3bfbb71f0672 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-with-tcp-route-and-inline-certificate.latest.golden @@ -0,0 +1,60 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "service:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "ingress_upstream_certificate", + "cluster": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICljCCAX4CCQCQMDsYO8FrPjANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV\nUzAeFw0yMjEyMjAxNzUwMjVaFw0yNzEyMTkxNzUwMjVaMA0xCzAJBgNVBAYTAlVT\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx95Opa6t4lGEpiTUogEB\nptqOdam2ch4BHQGhNhX/MrDwwuZQhttBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2\njQlhqTodElkbsd5vWY8R/bxJWQSoNvVE12TlzECxGpJEiHt4W0r8pGffk+rvplji\nUyCfnT1kGF3znOSjK1hRMTn6RKWCyYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409\ng9X5VU88/Bmmrz4cMyxce86Kc2ug5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftr\nXOvuCbO5IBRHMOBHiHTZ4rtGuhMaIr21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+W\nmQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBfCqoUIdPf/HGSbOorPyZWbyizNtHJ\nGL7x9cAeIYxpI5Y/WcO1o5v94lvrgm3FNfJoGKbV66+JxOge731FrfMpHplhar1Z\nRahYIzNLRBTLrwadLAZkApUpZvB8qDK4knsTWFYujNsylCww2A6ajzIMFNU4GkUK\nNtyHRuD+KYRmjXtyX1yHNqfGN3vOQmwavHq2R8wHYuBSc6LAHHV9vG+j0VsgMELO\nqwxn8SmLkSKbf2+MsQVzLCXXN5u+D8Yv+4py+oKP4EQ5aFZuDEx+r/G/31rTthww\nAAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAx95Opa6t4lGEpiTUogEBptqOdam2ch4BHQGhNhX/MrDwwuZQ\nhttBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2jQlhqTodElkbsd5vWY8R/bxJWQSo\nNvVE12TlzECxGpJEiHt4W0r8pGffk+rvpljiUyCfnT1kGF3znOSjK1hRMTn6RKWC\nyYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409g9X5VU88/Bmmrz4cMyxce86Kc2ug\n5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftrXOvuCbO5IBRHMOBHiHTZ4rtGuhMa\nIr21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+WmQIDAQABAoIBACYvceUzp2MK4gYA\nGWPOP2uKbBdM0l+hHeNV0WAM+dHMfmMuL4pkT36ucqt0ySOLjw6rQyOZG5nmA6t9\nsv0g4ae2eCMlyDIeNi1Yavu4Wt6YX4cTXbQKThm83C6W2X9THKbauBbxD621bsDK\n7PhiGPN60yPue7YwFQAPqqD4YaK+s22HFIzk9gwM/rkvAUNwRv7SyHMiFe4Igc1C\nEev7iHWzvj5Heoz6XfF+XNF9DU+TieSUAdjd56VyUb8XL4+uBTOhHwLiXvAmfaMR\nHvpcxeKnYZusS6NaOxcUHiJnsLNWrxmJj9WEGgQzuLxcLjTe4vVmELVZD8t3QUKj\nPAxu8tUCgYEA7KIWVn9dfVpokReorFym+J8FzLwSktP9RZYEMonJo00i8aii3K9s\nu/aSwRWQSCzmON1ZcxZzWhwQF9usz6kGCk//9+4hlVW90GtNK0RD+j7sp4aT2JI8\n9eLEjTG+xSXa7XWe98QncjjL9lu/yrRncSTxHs13q/XP198nn2aYuQ8CgYEA2Dnt\nsRBzv0fFEvzzFv7G/5f85mouN38TUYvxNRTjBLCXl9DeKjDkOVZ2b6qlfQnYXIru\nH+W+v+AZEb6fySXc8FRab7lkgTMrwE+aeI4rkW7asVwtclv01QJ5wMnyT84AgDD/\nDgt/RThFaHgtU9TW5GOZveL+l9fVPn7vKFdTJdcCgYEArJ99zjHxwJ1whNAOk1av\n09UmRPm6TvRo4heTDk8oEoIWCNatoHI0z1YMLuENNSnT9Q280FFDayvnrY/qnD7A\nkktT/sjwJOG8q8trKzIMqQS4XWm2dxoPcIyyOBJfCbEY6XuRsUuePxwh5qF942EB\nyS9a2s6nC4Ix0lgPrqAIr48CgYBgS/Q6riwOXSU8nqCYdiEkBYlhCJrKpnJxF9T1\nofa0yPzKZP/8ZEfP7VzTwHjxJehQ1qLUW9pG08P2biH1UEKEWdzo8vT6wVJT1F/k\nHtTycR8+a+Hlk2SHVRHqNUYQGpuIe8mrdJ1as4Pd0d/F/P0zO9Rlh+mAsGPM8HUM\nT0+9gwKBgHDoerX7NTskg0H0t8O+iSMevdxpEWp34ZYa9gHiftTQGyrRgERCa7Gj\nnZPAxKb2JoWyfnu3v7G5gZ8fhDFsiOxLbZv6UZJBbUIh1MjJISpXrForDrC2QNLX\nkHrHfwBFDB3KMudhQknsJzEJKCL/KmFH6o0MvsoaT9yzEl3K+ah/\n-----END RSA PRIVATE KEY-----\n" + } + } + ] + }, + "requireClientCertificate": false + } + } + } + ], + "listenerFilters": [ + { + "name": "envoy.filters.listener.tls_inspector", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector" + } + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway.latest.golden b/agent/xds/testdata/listeners/api-gateway.latest.golden new file mode 100644 index 000000000000..d2d839adf956 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/api-gateway-with-tcp-route-and-inline-certificate.latest.golden b/agent/xds/testdata/routes/api-gateway-with-tcp-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..306f5220e7b9 --- /dev/null +++ b/agent/xds/testdata/routes/api-gateway-with-tcp-route-and-inline-certificate.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/api-gateway-with-tcp-route-and-inline-certificate.latest.golden b/agent/xds/testdata/secrets/api-gateway-with-tcp-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/api-gateway-with-tcp-route-and-inline-certificate.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/connect-proxy-exported-to-peers.latest.golden b/agent/xds/testdata/secrets/connect-proxy-exported-to-peers.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/connect-proxy-exported-to-peers.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/connect-proxy-with-peered-upstreams.latest.golden b/agent/xds/testdata/secrets/connect-proxy-with-peered-upstreams.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/connect-proxy-with-peered-upstreams.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/defaults.latest.golden b/agent/xds/testdata/secrets/defaults.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/defaults.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/local-mesh-gateway-with-peered-upstreams.latest.golden b/agent/xds/testdata/secrets/local-mesh-gateway-with-peered-upstreams.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/local-mesh-gateway-with-peered-upstreams.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-peering-control-plane.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-peering-control-plane.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-peering-control-plane.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http-with-router.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http-with-router.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http-with-router.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services-http.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-exported-peered-services.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-imported-peered-services.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-imported-peered-services.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-imported-peered-services.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-peer-through-mesh-gateway-enabled.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-peer-through-mesh-gateway-enabled.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-peer-through-mesh-gateway-enabled.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-destination-http.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-destination-http.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-destination-http.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-destination.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-destination.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-destination.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-terminating-gateway-destinations-only.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-terminating-gateway-destinations-only.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-terminating-gateway-destinations-only.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-with-peered-upstreams.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-with-peered-upstreams.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-with-peered-upstreams.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy.latest.golden b/agent/xds/testdata/secrets/transparent-proxy.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/capture.sh b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/capture.sh new file mode 100644 index 000000000000..8ba0e0ddabc6 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/capture.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 api-gateway primary || true \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/service_gateway.hcl b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/service_gateway.hcl new file mode 100644 index 000000000000..486c25c59e5f --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/service_gateway.hcl @@ -0,0 +1,4 @@ +services { + name = "api-gateway" + kind = "api-gateway" +} diff --git a/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/setup.sh b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/setup.sh new file mode 100644 index 000000000000..2b2cbd160fa6 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-tls-overlapping-hosts/setup.sh @@ -0,0 +1,287 @@ +#!/bin/bash + +set -euo pipefail + +upsert_config_entry primary ' +kind = "api-gateway" +name = "api-gateway" +listeners = [ + { + name = "listener-one" + port = 9999 + protocol = "http" + tls = { + certificates = [ + { + kind = "inline-certificate" + name = "host-consul-example" + } + ] + } + }, + { + name = "listener-two" + port = 9998 + protocol = "http" + tls = { + certificates = [ + { + kind = "inline-certificate" + name = "host-consul-example" + }, + { + kind = "inline-certificate" + name = "also-host-consul-example" + }, + { + kind = "inline-certificate" + name = "other-consul-example" + } + ] + } + } +] +' + +upsert_config_entry primary ' +kind = "inline-certificate" +name = "host-consul-example" +private_key = </dev/null) + + echo "WANT CN: ${CN} (SNI: ${SERVER_NAME})" + echo "GOT CERT:" + echo "$CERT" + + echo "$CERT" | grep "CN = ${CN}" +} + function assert_envoy_version { local ADMINPORT=$1 run retry_default curl -f -s localhost:$ADMINPORT/server_info From e4e7c69f4469f7f1e4133985b6de3b24a9991c16 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 17 Feb 2023 15:47:38 -0500 Subject: [PATCH 009/421] Fix hostname alignment checks for HTTPRoutes (#16315) --- agent/consul/discoverychain/gateway.go | 13 +- agent/consul/discoverychain/gateway_test.go | 3 + agent/consul/gateways/controller_gateways.go | 8 + agent/proxycfg/snapshot.go | 15 +- agent/structs/config_entry_gateways.go | 7 + agent/structs/config_entry_routes.go | 26 +++ .../capture.sh | 3 + .../service_gateway.hcl | 4 + .../case-api-gateway-http-hostnames/setup.sh | 156 ++++++++++++++++++ .../case-api-gateway-http-hostnames/vars.sh | 3 + .../verify.bats | 66 ++++++++ 11 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 test/integration/connect/envoy/case-api-gateway-http-hostnames/capture.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-hostnames/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-api-gateway-http-hostnames/setup.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-hostnames/vars.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-hostnames/verify.bats diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index cd582f1ec02c..c9acdbbfda2f 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -17,6 +17,7 @@ type GatewayChainSynthesizer struct { trustDomain string suffix string gateway *structs.APIGatewayConfigEntry + hostname string matchesByHostname map[string][]hostnameMatch tcpRoutes []structs.TCPRouteConfigEntry } @@ -44,17 +45,17 @@ func (l *GatewayChainSynthesizer) AddTCPRoute(route structs.TCPRouteConfigEntry) l.tcpRoutes = append(l.tcpRoutes, route) } +// SetHostname sets the base hostname for a listener that this is being synthesized for +func (l *GatewayChainSynthesizer) SetHostname(hostname string) { + l.hostname = hostname +} + // AddHTTPRoute takes a new route and flattens its rule matches out per hostname. // This is required since a single route can specify multiple hostnames, and a // single hostname can be specified in multiple routes. Routing for a given // hostname must behave based on the aggregate of all rules that apply to it. func (l *GatewayChainSynthesizer) AddHTTPRoute(route structs.HTTPRouteConfigEntry) { - hostnames := route.Hostnames - if len(route.Hostnames) == 0 { - // add a wildcard if there are no explicit hostnames set - hostnames = append(hostnames, "*") - } - + hostnames := route.FilteredHostnames(l.hostname) for _, host := range hostnames { matches, ok := l.matchesByHostname[host] if !ok { diff --git a/agent/consul/discoverychain/gateway_test.go b/agent/consul/discoverychain/gateway_test.go index 1d6ec78d24c5..1c44f680b7c0 100644 --- a/agent/consul/discoverychain/gateway_test.go +++ b/agent/consul/discoverychain/gateway_test.go @@ -459,6 +459,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) { gatewayChainSynthesizer := NewGatewayChainSynthesizer(datacenter, "domain", "suffix", gateway) + gatewayChainSynthesizer.SetHostname("*") gatewayChainSynthesizer.AddHTTPRoute(tc.route) require.Equal(t, tc.expectedMatchesByHostname, gatewayChainSynthesizer.matchesByHostname) @@ -621,6 +622,8 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { + tc.synthesizer.SetHostname("*") + for _, tcpRoute := range tc.tcpRoutes { tc.synthesizer.AddTCPRoute(*tcpRoute) } diff --git a/agent/consul/gateways/controller_gateways.go b/agent/consul/gateways/controller_gateways.go index 53d8c9a888c5..f06b48581fa0 100644 --- a/agent/consul/gateways/controller_gateways.go +++ b/agent/consul/gateways/controller_gateways.go @@ -701,6 +701,14 @@ func (g *gatewayMeta) bindRoute(listener *structs.APIGatewayListener, bound *str return false, nil } + if route, ok := route.(*structs.HTTPRouteConfigEntry); ok { + // check our hostnames + hostnames := route.FilteredHostnames(listener.GetHostname()) + if len(hostnames) == 0 { + return false, fmt.Errorf("failed to bind route to gateway %s: listener %s is does not have any hostnames that match the route", route.GetName(), g.Gateway.Name) + } + } + if listener.Protocol == route.GetProtocol() && bound.BindRoute(structs.ResourceReference{ Kind: route.GetKind(), Name: route.GetName(), diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 6ce5a30083eb..f60e62319e5e 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -774,7 +774,7 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI } // Create a synthesized discovery chain for each service. - services, upstreams, compiled, err := c.synthesizeChains(datacenter, listener.Protocol, listener.Port, listener.Name, boundListener) + services, upstreams, compiled, err := c.synthesizeChains(datacenter, listener, boundListener) if err != nil { return configSnapshotIngressGateway{}, err } @@ -836,7 +836,7 @@ func (c *configSnapshotAPIGateway) ToIngress(datacenter string) (configSnapshotI }, nil } -func (c *configSnapshotAPIGateway) synthesizeChains(datacenter string, protocol structs.APIGatewayListenerProtocol, port int, name string, boundListener structs.BoundAPIGatewayListener) ([]structs.IngressService, structs.Upstreams, []*structs.CompiledDiscoveryChain, error) { +func (c *configSnapshotAPIGateway) synthesizeChains(datacenter string, listener structs.APIGatewayListener, boundListener structs.BoundAPIGatewayListener) ([]structs.IngressService, structs.Upstreams, []*structs.CompiledDiscoveryChain, error) { chains := []*structs.CompiledDiscoveryChain{} trustDomain := "" @@ -852,12 +852,13 @@ DOMAIN_LOOP: } } - synthesizer := discoverychain.NewGatewayChainSynthesizer(datacenter, trustDomain, name, c.GatewayConfig) + synthesizer := discoverychain.NewGatewayChainSynthesizer(datacenter, trustDomain, listener.Name, c.GatewayConfig) + synthesizer.SetHostname(listener.GetHostname()) for _, routeRef := range boundListener.Routes { switch routeRef.Kind { case structs.HTTPRoute: route, ok := c.HTTPRoutes.Get(routeRef) - if !ok || protocol != structs.ListenerProtocolHTTP { + if !ok || listener.Protocol != structs.ListenerProtocolHTTP { continue } synthesizer.AddHTTPRoute(*route) @@ -869,7 +870,7 @@ DOMAIN_LOOP: } case structs.TCPRoute: route, ok := c.TCPRoutes.Get(routeRef) - if !ok || protocol != structs.ListenerProtocolTCP { + if !ok || listener.Protocol != structs.ListenerProtocolTCP { continue } synthesizer.AddTCPRoute(*route) @@ -901,9 +902,9 @@ DOMAIN_LOOP: DestinationNamespace: service.NamespaceOrDefault(), DestinationPartition: service.PartitionOrDefault(), IngressHosts: service.Hosts, - LocalBindPort: port, + LocalBindPort: listener.Port, Config: map[string]interface{}{ - "protocol": string(protocol), + "protocol": string(listener.Protocol), }, }) } diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 83bbcfb15959..7d4ffd4479d0 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -897,6 +897,13 @@ type APIGatewayListener struct { TLS APIGatewayTLSConfiguration } +func (l APIGatewayListener) GetHostname() string { + if l.Hostname != "" { + return l.Hostname + } + return "*" +} + // APIGatewayTLSConfiguration specifies the configuration of a listener’s // TLS settings. type APIGatewayTLSConfiguration struct { diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 7c959d79b0b9..26e50b3f2d28 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -2,6 +2,7 @@ package structs import ( "fmt" + "strings" "github.com/hashicorp/consul/acl" ) @@ -121,6 +122,31 @@ func (e *HTTPRouteConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } +func (e *HTTPRouteConfigEntry) FilteredHostnames(listenerHostname string) []string { + if len(e.Hostnames) == 0 { + // we have no hostnames specified here, so treat it like a wildcard + return []string{listenerHostname} + } + + wildcardHostname := strings.ContainsRune(listenerHostname, '*') || listenerHostname == "*" + listenerHostname = strings.TrimPrefix(strings.TrimPrefix(listenerHostname, "*"), ".") + + hostnames := []string{} + for _, hostname := range e.Hostnames { + if wildcardHostname { + if strings.HasSuffix(hostname, listenerHostname) { + hostnames = append(hostnames, hostname) + } + continue + } + + if hostname == listenerHostname { + hostnames = append(hostnames, hostname) + } + } + return hostnames +} + // HTTPMatch specifies the criteria that should be // used in determining whether or not a request should // be routed to a given set of services. diff --git a/test/integration/connect/envoy/case-api-gateway-http-hostnames/capture.sh b/test/integration/connect/envoy/case-api-gateway-http-hostnames/capture.sh new file mode 100644 index 000000000000..8ba0e0ddabc6 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-hostnames/capture.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 api-gateway primary || true \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-hostnames/service_gateway.hcl b/test/integration/connect/envoy/case-api-gateway-http-hostnames/service_gateway.hcl new file mode 100644 index 000000000000..486c25c59e5f --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-hostnames/service_gateway.hcl @@ -0,0 +1,4 @@ +services { + name = "api-gateway" + kind = "api-gateway" +} diff --git a/test/integration/connect/envoy/case-api-gateway-http-hostnames/setup.sh b/test/integration/connect/envoy/case-api-gateway-http-hostnames/setup.sh new file mode 100644 index 000000000000..f4963a2a24fa --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-hostnames/setup.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +set -euo pipefail + +upsert_config_entry primary ' +kind = "api-gateway" +name = "api-gateway" +listeners = [ + { + name = "listener-one" + port = 9999 + protocol = "http" + hostname = "*.consul.example" + }, + { + name = "listener-two" + port = 9998 + protocol = "http" + hostname = "foo.bar.baz" + }, + { + name = "listener-three" + port = 9997 + protocol = "http" + hostname = "*.consul.example" + }, + { + name = "listener-four" + port = 9996 + protocol = "http" + hostname = "*.consul.example" + }, + { + name = "listener-five" + port = 9995 + protocol = "http" + hostname = "foo.bar.baz" + } +] +' + +upsert_config_entry primary ' +Kind = "proxy-defaults" +Name = "global" +Config { + protocol = "http" +} +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-one" +hostnames = ["test.consul.example"] +rules = [ + { + services = [ + { + name = "s1" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-one" + }, +] +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-two" +hostnames = ["foo.bar.baz"] +rules = [ + { + services = [ + { + name = "s1" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-two" + }, +] +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-three" +hostnames = ["foo.bar.baz"] +rules = [ + { + services = [ + { + name = "s1" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-three" + }, +] +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-four" +rules = [ + { + services = [ + { + name = "s1" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-four" + }, +] +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-five" +rules = [ + { + services = [ + { + name = "s1" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-five" + }, +] +' + +register_services primary + +gen_envoy_bootstrap api-gateway 20000 primary true +gen_envoy_bootstrap s1 19000 \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-hostnames/vars.sh b/test/integration/connect/envoy/case-api-gateway-http-hostnames/vars.sh new file mode 100644 index 000000000000..38a47d852783 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-hostnames/vars.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +export REQUIRED_SERVICES="$DEFAULT_REQUIRED_SERVICES api-gateway-primary" diff --git a/test/integration/connect/envoy/case-api-gateway-http-hostnames/verify.bats b/test/integration/connect/envoy/case-api-gateway-http-hostnames/verify.bats new file mode 100644 index 000000000000..ba109ea6f9dd --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-hostnames/verify.bats @@ -0,0 +1,66 @@ +#!/usr/bin/env bats + +load helpers + +@test "api gateway proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "api gateway should have be accepted and not conflicted" { + assert_config_entry_status Accepted True Accepted primary api-gateway api-gateway + assert_config_entry_status Conflicted False NoConflict primary api-gateway api-gateway +} + +@test "api gateway should be bound to route one" { + assert_config_entry_status Bound True Bound primary http-route api-gateway-route-one + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 +} + +@test "api gateway should be bound to route two" { + assert_config_entry_status Bound True Bound primary http-route api-gateway-route-two +} + +@test "api gateway should be unbound to route three" { + assert_config_entry_status Bound False FailedToBind primary http-route api-gateway-route-three +} + +@test "api gateway should be bound to route four" { + assert_config_entry_status Bound True Bound primary http-route api-gateway-route-four +} + +@test "api gateway should be bound to route five" { + assert_config_entry_status Bound True Bound primary http-route api-gateway-route-five +} + +@test "api gateway should be able to connect to s1 via route one with the proper host" { + run retry_long curl -H "Host: test.consul.example" -s -f -d hello localhost:9999 + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] +} + +@test "api gateway should not be able to connect to s1 via route one with a mismatched host" { + run retry_default sh -c "curl -H \"Host: foo.consul.example\" -sI -o /dev/null -w \"%{http_code}\" localhost:9999 | grep 404" + [ "$status" -eq 0 ] + [[ "$output" == "404" ]] +} + +@test "api gateway should be able to connect to s1 via route two with the proper host" { + run retry_long curl -H "Host: foo.bar.baz" -s -f -d hello localhost:9998 + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] +} + +@test "api gateway should be able to connect to s1 via route four with any subdomain of the listener host" { + run retry_long curl -H "Host: test.consul.example" -s -f -d hello localhost:9996 + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] + run retry_long curl -H "Host: foo.consul.example" -s -f -d hello localhost:9996 + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] +} + +@test "api gateway should be able to connect to s1 via route five with the proper host" { + run retry_long curl -H "Host: foo.bar.baz" -s -f -d hello localhost:9995 + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] +} \ No newline at end of file From 0e89df2b7a44614920390fed24c496477ef91aca Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Feb 2023 16:00:25 -0500 Subject: [PATCH 010/421] Backport of [OSS] security: update go to 1.20.1 into release/1.15.x (#16314) * backport of commit 2a8cdf1747ca0868c6d9892a56b3a3f444773946 * backport of commit 716d1620485cbc7c0f0522bfa297aec0aebba2f9 * backport of commit 2c757fac4f414b28d0c16df5be4a991071146ce8 * backport of commit f81c5d4933b9d22b9a14461572fa829fc9f8cd61 * backport of commit ff4f0596f3c17a55068d66bfef3389f1eb17731b * backport of commit 31bbca9fe4eb3cea81594e4e1ea182cb5f889d44 * backport of commit 2f541913535b3ce040a2becf6f4579916dcdc0f5 * backport of commit c263147180ffd3aea68fe813d44d40d5426e891d --------- Co-authored-by: DanStough --- .changelog/16263.txt | 4 +++ .circleci/config.yml | 26 +++++++------- .github/workflows/build.yml | 20 +++++------ GNUmakefile | 6 ++-- agent/agent_test.go | 5 +-- agent/consul/auto_config_endpoint_test.go | 5 ++- agent/consul/internal_endpoint_test.go | 2 +- agent/consul/leader_peering_test.go | 4 +-- agent/consul/state/acl_test.go | 2 -- agent/coordinate_endpoint_test.go | 6 ++-- agent/grpc-external/limiter/limiter_test.go | 4 --- agent/prepared_query_endpoint_test.go | 21 ++++++------ agent/testagent.go | 5 --- agent/txn_endpoint_test.go | 18 +++++----- api/go.mod | 2 +- build-support/docker/Build-Go.dockerfile | 2 +- command/members/members_test.go | 3 -- envoyextensions/go.mod | 2 +- go.mod | 2 +- lib/rand.go | 34 ------------------- main.go | 5 --- proto-public/pbdns/mock_DNSServiceClient.go | 7 ++-- proto-public/pbdns/mock_DNSServiceServer.go | 7 ++-- .../pbdns/mock_UnsafeDNSServiceServer.go | 2 +- sdk/freeport/freeport.go | 1 - sdk/go.mod | 2 +- test/integration/consul-container/go.mod | 2 +- tlsutil/config_test.go | 2 +- troubleshoot/go.mod | 2 +- 29 files changed, 80 insertions(+), 123 deletions(-) create mode 100644 .changelog/16263.txt delete mode 100644 lib/rand.go diff --git a/.changelog/16263.txt b/.changelog/16263.txt new file mode 100644 index 000000000000..a8cd3f9043af --- /dev/null +++ b/.changelog/16263.txt @@ -0,0 +1,4 @@ +```release-note:security +Upgrade to use Go 1.20.1. +This resolves vulnerabilities [CVE-2022-41724](https://go.dev/issue/58001) in `crypto/tls` and [CVE-2022-41723](https://go.dev/issue/57855) in `net/http`. +``` diff --git a/.circleci/config.yml b/.circleci/config.yml index dae806a625bd..d50ddb1fa72a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ references: GIT_COMMITTER_NAME: circleci-consul S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 BASH_ENV: .circleci/bash_env.sh - GO_VERSION: 1.19.4 + GO_VERSION: 1.20.1 envoy-versions: &supported_envoy_versions - &default_envoy_version "1.21.5" - "1.22.5" @@ -39,7 +39,7 @@ references: images: # When updating the Go version, remember to also update the versions in the # workflows section for go-test-lib jobs. - go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.19.4 + go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.20.1 ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02 cache: @@ -613,7 +613,7 @@ jobs: - run: *notify-slack-failure nomad-integration-test: &NOMAD_TESTS docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.19 + - image: docker.mirror.hashicorp.services/cimg/go:1.20 parameters: nomad-version: type: enum @@ -1110,34 +1110,34 @@ workflows: - go-test-lib: name: "go-test-envoyextensions" path: envoyextensions - go-version: "1.19" + go-version: "1.20" requires: [dev-build] <<: *filter-ignore-non-go-branches - go-test-lib: name: "go-test-troubleshoot" path: troubleshoot - go-version: "1.19" + go-version: "1.20" requires: [dev-build] <<: *filter-ignore-non-go-branches - go-test-lib: - name: "go-test-api go1.18" + name: "go-test-api go1.19" path: api - go-version: "1.18" + go-version: "1.19" requires: [dev-build] - go-test-lib: - name: "go-test-api go1.19" + name: "go-test-api go1.20" path: api - go-version: "1.19" + go-version: "1.20" requires: [dev-build] - go-test-lib: - name: "go-test-sdk go1.18" + name: "go-test-sdk go1.19" path: sdk - go-version: "1.18" + go-version: "1.19" <<: *filter-ignore-non-go-branches - go-test-lib: - name: "go-test-sdk go1.19" + name: "go-test-sdk go1.20" path: sdk - go-version: "1.19" + go-version: "1.20" <<: *filter-ignore-non-go-branches - go-test-race: *filter-ignore-non-go-branches - go-test-32bit: *filter-ignore-non-go-branches diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20f316c12e3a..b0c1dbbd82da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,15 +79,15 @@ jobs: strategy: matrix: include: - - {go: "1.19.4", goos: "linux", goarch: "386"} - - {go: "1.19.4", goos: "linux", goarch: "amd64"} - - {go: "1.19.4", goos: "linux", goarch: "arm"} - - {go: "1.19.4", goos: "linux", goarch: "arm64"} - - {go: "1.19.4", goos: "freebsd", goarch: "386"} - - {go: "1.19.4", goos: "freebsd", goarch: "amd64"} - - {go: "1.19.4", goos: "windows", goarch: "386"} - - {go: "1.19.4", goos: "windows", goarch: "amd64"} - - {go: "1.19.4", goos: "solaris", goarch: "amd64"} + - {go: "1.20.1", goos: "linux", goarch: "386"} + - {go: "1.20.1", goos: "linux", goarch: "amd64"} + - {go: "1.20.1", goos: "linux", goarch: "arm"} + - {go: "1.20.1", goos: "linux", goarch: "arm64"} + - {go: "1.20.1", goos: "freebsd", goarch: "386"} + - {go: "1.20.1", goos: "freebsd", goarch: "amd64"} + - {go: "1.20.1", goos: "windows", goarch: "386"} + - {go: "1.20.1", goos: "windows", goarch: "amd64"} + - {go: "1.20.1", goos: "solaris", goarch: "amd64"} fail-fast: true name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build @@ -176,7 +176,7 @@ jobs: matrix: goos: [ darwin ] goarch: [ "amd64", "arm64" ] - go: [ "1.19.4" ] + go: [ "1.20.1" ] fail-fast: true name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build diff --git a/GNUmakefile b/GNUmakefile index f1cebb689553..ee765693f4a9 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -7,11 +7,11 @@ SHELL = bash # These version variables can either be a valid string for "go install @" # or the string @DEV to imply use what is currently installed locally. ### -GOLANGCI_LINT_VERSION='v1.50.1' -MOCKERY_VERSION='v2.15.0' +GOLANGCI_LINT_VERSION='v1.51.1' +MOCKERY_VERSION='v2.20.0' BUF_VERSION='v1.4.0' PROTOC_GEN_GO_GRPC_VERSION="v1.2.0" -MOG_VERSION='v0.3.0' +MOG_VERSION='v0.4.0' PROTOC_GO_INJECT_TAG_VERSION='v1.3.0' PROTOC_GEN_GO_BINARY_VERSION="v0.1.0" DEEP_COPY_VERSION='bc3f5aa5735d8a54961580a3a24422c308c831c2' diff --git a/agent/agent_test.go b/agent/agent_test.go index d6dd2cc5fcfa..bcc57cf41e0c 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -4,12 +4,13 @@ import ( "bytes" "context" "crypto/md5" + "crypto/rand" "crypto/tls" "crypto/x509" "encoding/base64" "encoding/json" "fmt" - "math/rand" + mathrand "math/rand" "net" "net/http" "net/http/httptest" @@ -752,7 +753,7 @@ func testAgent_AddServices_AliasUpdateCheckNotReverted(t *testing.T, extraHCL st func test_createAlias(t *testing.T, agent *TestAgent, chk *structs.CheckType, expectedResult string) func(r *retry.R) { t.Helper() - serviceNum := rand.Int() + serviceNum := mathrand.Int() srv := &structs.NodeService{ Service: fmt.Sprintf("serviceAlias-%d", serviceNum), Tags: []string{"tag1"}, diff --git a/agent/consul/auto_config_endpoint_test.go b/agent/consul/auto_config_endpoint_test.go index ac9ea4128ddf..1f0f8e18a1ca 100644 --- a/agent/consul/auto_config_endpoint_test.go +++ b/agent/consul/auto_config_endpoint_test.go @@ -3,12 +3,11 @@ package consul import ( "bytes" "crypto" - crand "crypto/rand" + "crypto/rand" "crypto/x509" "encoding/base64" "encoding/pem" "fmt" - "math/rand" "net" "net/url" "os" @@ -884,7 +883,7 @@ func TestAutoConfig_parseAutoConfigCSR(t *testing.T) { // customizations to allow for better unit testing. createCSR := func(tmpl *x509.CertificateRequest, privateKey crypto.Signer) (string, error) { connect.HackSANExtensionForCSR(tmpl) - bs, err := x509.CreateCertificateRequest(crand.Reader, tmpl, privateKey) + bs, err := x509.CreateCertificateRequest(rand.Reader, tmpl, privateKey) require.NoError(t, err) var csrBuf bytes.Buffer err = pem.Encode(&csrBuf, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: bs}) diff --git a/agent/consul/internal_endpoint_test.go b/agent/consul/internal_endpoint_test.go index e0aa941b90e5..181de4ed82c2 100644 --- a/agent/consul/internal_endpoint_test.go +++ b/agent/consul/internal_endpoint_test.go @@ -1,9 +1,9 @@ package consul import ( + "crypto/rand" "encoding/base64" "fmt" - "math/rand" "os" "strings" "testing" diff --git a/agent/consul/leader_peering_test.go b/agent/consul/leader_peering_test.go index 4be6326c0959..143448535aa1 100644 --- a/agent/consul/leader_peering_test.go +++ b/agent/consul/leader_peering_test.go @@ -478,7 +478,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { t.Run("server-name-validation", func(t *testing.T) { testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) { token.ServerName = "wrong.name" - }, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`) + }, `transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`) }) t.Run("bad-ca-roots", func(t *testing.T) { wrongRoot, err := os.ReadFile("../../test/client_certs/rootca.crt") @@ -486,7 +486,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) { token.CA = []string{string(wrongRoot)} - }, `transport: authentication handshake failed: x509: certificate signed by unknown authority`) + }, `transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate signed by unknown authority`) }) } diff --git a/agent/consul/state/acl_test.go b/agent/consul/state/acl_test.go index 5e01514730fe..fc00728282af 100644 --- a/agent/consul/state/acl_test.go +++ b/agent/consul/state/acl_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/proto/pbacl" ) @@ -3570,7 +3569,6 @@ func TestStateStore_ACLPolicies_Snapshot_Restore(t *testing.T) { } func TestTokenPoliciesIndex(t *testing.T) { - lib.SeedMathRand() idIndex := &memdb.IndexSchema{ Name: "id", diff --git a/agent/coordinate_endpoint_test.go b/agent/coordinate_endpoint_test.go index 4782ae72d0f8..71492d909c42 100644 --- a/agent/coordinate_endpoint_test.go +++ b/agent/coordinate_endpoint_test.go @@ -40,9 +40,9 @@ func TestCoordinate_Disabled_Response(t *testing.T) { req, _ := http.NewRequest("PUT", "/should/not/care", nil) resp := httptest.NewRecorder() obj, err := tt(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 401 { - t.Fatalf("expected status 401 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 401 { + t.Fatalf("expected status 401 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) diff --git a/agent/grpc-external/limiter/limiter_test.go b/agent/grpc-external/limiter/limiter_test.go index cef6a4d41719..7f5b9654a0aa 100644 --- a/agent/grpc-external/limiter/limiter_test.go +++ b/agent/grpc-external/limiter/limiter_test.go @@ -8,12 +8,8 @@ import ( "time" "github.com/stretchr/testify/require" - - "github.com/hashicorp/consul/lib" ) -func init() { lib.SeedMathRand() } - func TestSessionLimiter(t *testing.T) { lim := NewSessionLimiter() diff --git a/agent/prepared_query_endpoint_test.go b/agent/prepared_query_endpoint_test.go index 33240fd77aed..09012bc2a06f 100644 --- a/agent/prepared_query_endpoint_test.go +++ b/agent/prepared_query_endpoint_test.go @@ -13,9 +13,10 @@ import ( "github.com/hashicorp/consul/testrpc" + "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/types" - "github.com/stretchr/testify/require" ) // MockPreparedQuery is a fake endpoint that we inject into the Consul server @@ -628,9 +629,9 @@ func TestPreparedQuery_Execute(t *testing.T) { req, _ := http.NewRequest("GET", "/v1/query/not-there/execute", body) resp := httptest.NewRecorder() _, err := a.srv.PreparedQuerySpecific(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 404 { - t.Fatalf("expected status 404 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 404 { + t.Fatalf("expected status 404 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) @@ -768,9 +769,9 @@ func TestPreparedQuery_Explain(t *testing.T) { req, _ := http.NewRequest("GET", "/v1/query/not-there/explain", body) resp := httptest.NewRecorder() _, err := a.srv.PreparedQuerySpecific(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 404 { - t.Fatalf("expected status 404 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 404 { + t.Fatalf("expected status 404 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) @@ -862,9 +863,9 @@ func TestPreparedQuery_Get(t *testing.T) { req, _ := http.NewRequest("GET", "/v1/query/f004177f-2c28-83b7-4229-eacc25fe55d1", body) resp := httptest.NewRecorder() _, err := a.srv.PreparedQuerySpecific(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 404 { - t.Fatalf("expected status 404 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 404 { + t.Fatalf("expected status 404 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) diff --git a/agent/testagent.go b/agent/testagent.go index db08be40a4b7..54db5c72ba42 100644 --- a/agent/testagent.go +++ b/agent/testagent.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "fmt" "io" - "math/rand" "net" "net/http/httptest" "path/filepath" @@ -32,10 +31,6 @@ import ( "github.com/hashicorp/consul/tlsutil" ) -func init() { - rand.Seed(time.Now().UnixNano()) // seed random number generator -} - // TestAgent encapsulates an Agent with a default configuration and // startup procedure suitable for testing. It panics if there are errors // during creation or startup instead of returning errors. It manages a diff --git a/agent/txn_endpoint_test.go b/agent/txn_endpoint_test.go index 90e5359955c0..ce94b5c3e63b 100644 --- a/agent/txn_endpoint_test.go +++ b/agent/txn_endpoint_test.go @@ -67,9 +67,9 @@ func TestTxnEndpoint_Bad_Size_Item(t *testing.T) { t.Fatalf("err: %v", err) } } else { - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 413 { - t.Fatalf("expected 413 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 413 { + t.Fatalf("expected 413 but got %d", httpErr.StatusCode) } } else { t.Fatalf("excected HTTP error but got %v", err) @@ -150,9 +150,9 @@ func TestTxnEndpoint_Bad_Size_Net(t *testing.T) { t.Fatalf("err: %v", err) } } else { - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 413 { - t.Fatalf("expected 413 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 413 { + t.Fatalf("expected 413 but got %d", httpErr.StatusCode) } } else { t.Fatalf("excected HTTP error but got %v", err) @@ -220,9 +220,9 @@ func TestTxnEndpoint_Bad_Size_Ops(t *testing.T) { resp := httptest.NewRecorder() _, err := a.srv.Txn(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 413 { - t.Fatalf("expected 413 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 413 { + t.Fatalf("expected 413 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) diff --git a/api/go.mod b/api/go.mod index 20c8e80814b2..d9a68569df7f 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/api -go 1.18 +go 1.20 replace github.com/hashicorp/consul/sdk => ../sdk diff --git a/build-support/docker/Build-Go.dockerfile b/build-support/docker/Build-Go.dockerfile index cd578b451b78..543344ea3f46 100644 --- a/build-support/docker/Build-Go.dockerfile +++ b/build-support/docker/Build-Go.dockerfile @@ -1,4 +1,4 @@ -ARG GOLANG_VERSION=1.19.2 +ARG GOLANG_VERSION=1.20.1 FROM golang:${GOLANG_VERSION} WORKDIR /consul diff --git a/command/members/members_test.go b/command/members/members_test.go index cc4a21742aec..c9a2d42b77d6 100644 --- a/command/members/members_test.go +++ b/command/members/members_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/consul/agent" consulapi "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/lib" ) // TODO(partitions): split these tests @@ -206,8 +205,6 @@ func zip(t *testing.T, k, v []string) map[string]string { } func TestSortByMemberNamePartitionAndSegment(t *testing.T) { - lib.SeedMathRand() - // For the test data we'll give them names that would sort them backwards // if we only sorted by name. newData := func() []*consulapi.AgentMember { diff --git a/envoyextensions/go.mod b/envoyextensions/go.mod index f96560804c9d..5a73e969c2db 100644 --- a/envoyextensions/go.mod +++ b/envoyextensions/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/envoyextensions -go 1.19 +go 1.20 replace github.com/hashicorp/consul/api => ../api diff --git a/go.mod b/go.mod index 5a38efd9dc8d..299e682647c0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul -go 1.19 +go 1.20 replace ( github.com/hashicorp/consul/api => ./api diff --git a/lib/rand.go b/lib/rand.go deleted file mode 100644 index 22aa4f3544bb..000000000000 --- a/lib/rand.go +++ /dev/null @@ -1,34 +0,0 @@ -package lib - -import ( - crand "crypto/rand" - "math" - "math/big" - "math/rand" - "sync" - "time" -) - -var ( - once sync.Once - - // SeededSecurely is set to true if a cryptographically secure seed - // was used to initialize rand. When false, the start time is used - // as a seed. - SeededSecurely bool -) - -// SeedMathRand provides weak, but guaranteed seeding, which is better than -// running with Go's default seed of 1. A call to SeedMathRand() is expected -// to be called via init(), but never a second time. -func SeedMathRand() { - once.Do(func() { - n, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - rand.Seed(time.Now().UTC().UnixNano()) - return - } - rand.Seed(n.Int64()) - SeededSecurely = true - }) -} diff --git a/main.go b/main.go index 5138f8c2219d..804635060a81 100644 --- a/main.go +++ b/main.go @@ -11,14 +11,9 @@ import ( "github.com/hashicorp/consul/command" "github.com/hashicorp/consul/command/cli" "github.com/hashicorp/consul/command/version" - "github.com/hashicorp/consul/lib" _ "github.com/hashicorp/consul/service_os" ) -func init() { - lib.SeedMathRand() -} - func main() { os.Exit(realMain()) } diff --git a/proto-public/pbdns/mock_DNSServiceClient.go b/proto-public/pbdns/mock_DNSServiceClient.go index 24906ab8547a..d9fffda65aef 100644 --- a/proto-public/pbdns/mock_DNSServiceClient.go +++ b/proto-public/pbdns/mock_DNSServiceClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.15.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package pbdns @@ -27,6 +27,10 @@ func (_m *MockDNSServiceClient) Query(ctx context.Context, in *QueryRequest, opt ret := _m.Called(_ca...) var r0 *QueryResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *QueryRequest, ...grpc.CallOption) (*QueryResponse, error)); ok { + return rf(ctx, in, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, *QueryRequest, ...grpc.CallOption) *QueryResponse); ok { r0 = rf(ctx, in, opts...) } else { @@ -35,7 +39,6 @@ func (_m *MockDNSServiceClient) Query(ctx context.Context, in *QueryRequest, opt } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *QueryRequest, ...grpc.CallOption) error); ok { r1 = rf(ctx, in, opts...) } else { diff --git a/proto-public/pbdns/mock_DNSServiceServer.go b/proto-public/pbdns/mock_DNSServiceServer.go index e9bd338daf12..e78c7d4c304b 100644 --- a/proto-public/pbdns/mock_DNSServiceServer.go +++ b/proto-public/pbdns/mock_DNSServiceServer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.15.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package pbdns @@ -18,6 +18,10 @@ func (_m *MockDNSServiceServer) Query(_a0 context.Context, _a1 *QueryRequest) (* ret := _m.Called(_a0, _a1) var r0 *QueryResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *QueryRequest) (*QueryResponse, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, *QueryRequest) *QueryResponse); ok { r0 = rf(_a0, _a1) } else { @@ -26,7 +30,6 @@ func (_m *MockDNSServiceServer) Query(_a0 context.Context, _a1 *QueryRequest) (* } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *QueryRequest) error); ok { r1 = rf(_a0, _a1) } else { diff --git a/proto-public/pbdns/mock_UnsafeDNSServiceServer.go b/proto-public/pbdns/mock_UnsafeDNSServiceServer.go index 0a6c47c2cb7b..43a9e1e461ae 100644 --- a/proto-public/pbdns/mock_UnsafeDNSServiceServer.go +++ b/proto-public/pbdns/mock_UnsafeDNSServiceServer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.15.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package pbdns diff --git a/sdk/freeport/freeport.go b/sdk/freeport/freeport.go index 6eda1d4279b9..6c275fe86674 100644 --- a/sdk/freeport/freeport.go +++ b/sdk/freeport/freeport.go @@ -114,7 +114,6 @@ func initialize() { panic("freeport: block size too big or too many blocks requested") } - rand.Seed(time.Now().UnixNano()) firstPort, lockLn = alloc() condNotEmpty = sync.NewCond(&mu) diff --git a/sdk/go.mod b/sdk/go.mod index b7c2eb014260..63ad3671a3e1 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/sdk -go 1.18 +go 1.20 require ( github.com/hashicorp/go-cleanhttp v0.5.1 diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index a88a6aff0b3b..513612707d7c 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/test/integration/consul-container -go 1.19 +go 1.20 require ( github.com/avast/retry-go v3.0.0+incompatible diff --git a/tlsutil/config_test.go b/tlsutil/config_test.go index 7ce7893bfbe6..388b4934ed4d 100644 --- a/tlsutil/config_test.go +++ b/tlsutil/config_test.go @@ -906,7 +906,7 @@ func TestConfigurator_outgoingWrapperALPN_serverHasNoNodeNameInSAN(t *testing.T) _, err = wrap("dc1", "bob", "foo", client) require.Error(t, err) - _, ok := err.(x509.HostnameError) + _, ok := err.(*tls.CertificateVerificationError) require.True(t, ok) client.Close() diff --git a/troubleshoot/go.mod b/troubleshoot/go.mod index c4e445ee05f1..cf82c88f1e1d 100644 --- a/troubleshoot/go.mod +++ b/troubleshoot/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/troubleshoot -go 1.19 +go 1.20 replace github.com/hashicorp/consul/api => ../api From 9ce2cff6002dc38bc7ab154f9a5cc54f753dcac0 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 17 Feb 2023 16:03:30 -0500 Subject: [PATCH 011/421] Backport of Add stricter validation and some normalization code for API Gateway ConfigEntries into release/1.15.x (#16317) * Add stricter validation and some normalization code for API Gateway ConfigEntries * Switch to certs with valid hostnames in them --- agent/structs/config_entry_gateways.go | 95 ++--- .../config_entry_inline_certificate.go | 17 + .../config_entry_inline_certificate_test.go | 96 ++++- agent/structs/config_entry_routes.go | 337 ++++++++++++------ agent/structs/config_entry_routes_test.go | 206 +++++++++++ api/config_entry_inline_certificate_test.go | 82 +++-- 6 files changed, 596 insertions(+), 237 deletions(-) diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 7d4ffd4479d0..0d4019896f3a 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -713,6 +713,18 @@ type APIGatewayConfigEntry struct { RaftIndex } +func (e *APIGatewayConfigEntry) GetKind() string { return APIGateway } +func (e *APIGatewayConfigEntry) GetName() string { return e.Name } +func (e *APIGatewayConfigEntry) GetMeta() map[string]string { return e.Meta } +func (e *APIGatewayConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } +func (e *APIGatewayConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { return &e.EnterpriseMeta } + +var _ ControlledConfigEntry = (*APIGatewayConfigEntry)(nil) + +func (e *APIGatewayConfigEntry) GetStatus() Status { return e.Status } +func (e *APIGatewayConfigEntry) SetStatus(status Status) { e.Status = status } +func (e *APIGatewayConfigEntry) DefaultStatus() Status { return Status{} } + func (e *APIGatewayConfigEntry) ListenerIsReady(name string) bool { for _, condition := range e.Status.Conditions { if !condition.Resource.IsSame(&ResourceReference{ @@ -732,34 +744,28 @@ func (e *APIGatewayConfigEntry) ListenerIsReady(name string) bool { return true } -func (e *APIGatewayConfigEntry) GetKind() string { - return APIGateway -} - -func (e *APIGatewayConfigEntry) GetName() string { - if e == nil { - return "" - } - return e.Name -} - -func (e *APIGatewayConfigEntry) GetMeta() map[string]string { - if e == nil { - return nil - } - return e.Meta -} - func (e *APIGatewayConfigEntry) Normalize() error { for i, listener := range e.Listeners { protocol := strings.ToLower(string(listener.Protocol)) listener.Protocol = APIGatewayListenerProtocol(protocol) e.Listeners[i] = listener + + for i, cert := range listener.TLS.Certificates { + if cert.Kind == "" { + cert.Kind = InlineCertificate + } + listener.TLS.Certificates[i] = cert + } } + return nil } func (e *APIGatewayConfigEntry) Validate() error { + if err := validateConfigEntryMeta(e.Meta); err != nil { + return err + } + if err := e.validateListenerNames(); err != nil { return err } @@ -843,34 +849,6 @@ func (e *APIGatewayConfigEntry) CanWrite(authz acl.Authorizer) error { return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext) } -func (e *APIGatewayConfigEntry) GetRaftIndex() *RaftIndex { - if e == nil { - return &RaftIndex{} - } - return &e.RaftIndex -} - -func (e *APIGatewayConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { - if e == nil { - return nil - } - return &e.EnterpriseMeta -} - -var _ ControlledConfigEntry = (*APIGatewayConfigEntry)(nil) - -func (e *APIGatewayConfigEntry) GetStatus() Status { - return e.Status -} - -func (e *APIGatewayConfigEntry) SetStatus(status Status) { - e.Status = status -} - -func (e *APIGatewayConfigEntry) DefaultStatus() Status { - return Status{} -} - // APIGatewayListenerProtocol is the protocol that an APIGateway listener uses type APIGatewayListenerProtocol string @@ -991,27 +969,10 @@ func (e *BoundAPIGatewayConfigEntry) IsInitializedForGateway(gateway *APIGateway return true } -func (e *BoundAPIGatewayConfigEntry) GetKind() string { - return BoundAPIGateway -} - -func (e *BoundAPIGatewayConfigEntry) GetName() string { - if e == nil { - return "" - } - return e.Name -} - -func (e *BoundAPIGatewayConfigEntry) GetMeta() map[string]string { - if e == nil { - return nil - } - return e.Meta -} - -func (e *BoundAPIGatewayConfigEntry) Normalize() error { - return nil -} +func (e *BoundAPIGatewayConfigEntry) GetKind() string { return BoundAPIGateway } +func (e *BoundAPIGatewayConfigEntry) GetName() string { return e.Name } +func (e *BoundAPIGatewayConfigEntry) GetMeta() map[string]string { return e.Meta } +func (e *BoundAPIGatewayConfigEntry) Normalize() error { return nil } func (e *BoundAPIGatewayConfigEntry) Validate() error { allowedCertificateKinds := map[string]bool{ diff --git a/agent/structs/config_entry_inline_certificate.go b/agent/structs/config_entry_inline_certificate.go index 18e3c7716db3..bdbcbc6ae05a 100644 --- a/agent/structs/config_entry_inline_certificate.go +++ b/agent/structs/config_entry_inline_certificate.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/hashicorp/consul/acl" + "github.com/miekg/dns" ) // InlineCertificateConfigEntry manages the configuration for an inline certificate @@ -39,6 +40,10 @@ func (e *InlineCertificateConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { func (e *InlineCertificateConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } func (e *InlineCertificateConfigEntry) Validate() error { + if err := validateConfigEntryMeta(e.Meta); err != nil { + return err + } + privateKeyBlock, _ := pem.Decode([]byte(e.PrivateKey)) if privateKeyBlock == nil { return errors.New("failed to parse private key PEM") @@ -61,6 +66,18 @@ func (e *InlineCertificateConfigEntry) Validate() error { return err } + // validate that each host referenced in the CN, DNSSans, and IPSans + // are valid hostnames + hosts, err := e.Hosts() + if err != nil { + return err + } + for _, host := range hosts { + if _, ok := dns.IsDomainName(host); !ok { + return fmt.Errorf("host %q must be a valid DNS hostname", host) + } + } + return nil } diff --git a/agent/structs/config_entry_inline_certificate_test.go b/agent/structs/config_entry_inline_certificate_test.go index 3e9d84e4f4bc..db46ca92a7df 100644 --- a/agent/structs/config_entry_inline_certificate_test.go +++ b/agent/structs/config_entry_inline_certificate_test.go @@ -5,6 +5,73 @@ import "testing" const ( // generated via openssl req -x509 -sha256 -days 1825 -newkey rsa:2048 -keyout private.key -out certificate.crt validPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA0wzZeonUklhOvJ0AxcdDdCTiMwR9tsm/6IGcw9Jm50xVY+qg +5GFg1RWrQaODq7Gjqd/JDUAwtTBnQMs1yt6nbsHe2QhbD4XeqtZ+6fTv1ZpG3k8F +eB/M01xFqovczRV/ie77wd4vqoPD+AcfD8NDAFJt3htwUgGIqkQHP329Sh3TtLga +9ZMCs1MoTT+POYGUPL8bwt9R6ClNrucbH4Bs6OnX2ZFbKF75O9OHKNxWTmpDSodv +OFbFyKps3BfnPuF0Z6mj5M5yZeCjmtfS25PrsM3pMBGK5YHb0MlFfZIrIGboMbrz +9F/BMQJ64pMe43KwqHvTnbKWhp6PzLhEkPGLnwIDAQABAoIBADBEJAiONPszDu67 +yU1yAM8zEDgysr127liyK7PtDnOfVXgAVMNmMcsJpZzhVF+TxKY487YAFCOb6kE7 +OBYpTYla9SgVbR3js8TGQUgoKCFlowd8cvfB7gn4dEZIrjqIzB4zdYgk1Cne8JZs +qoHkWhJcx5ugEtPuXd7yp+WxT/T+6uOro06scp67NhP5t9yoAGFv5Vdb577RuzRo +Wkd9higQ9A20+GtjCY0EYxdgRviWvW7mM5/F+Lzcaui86ME+ga754gX8zgW3+NJ5 +LMsz5OLSnh291Uyjmr77HWBv/xvpq01Fls0LyJcgxFVZuJs5GQz+l3otSqv4FTP6 +Ua9w/YECgYEA8To3dgUK1QhzX5rwhWtlst3pItGTvmEdNzXmjgSylu7uKM13i+xg +llhp2uXrOEtuL+xtBZdeFNaijusbyqjg0xj6e4o31c19okuuDkJD5/sfQq22bvrn +gVJMGuESprIiPePrEyrXCHOdxH6eDgR2dIzAeO5vz0nnKGFAWrJJbvECgYEA3/mJ +eacXOJznw4Sa8jGWS2FtZLKxDHph7uDKMJmuG0ukb3aHJ9dMHrPleCLo8mhpoObA +hueoIbIP7swGrQx79+nZbnQpF6rMp6FAU5bF3gSrj1eWbaeh8pn9mrv4hal9USmn +orTbXMxDp3XSh7voR8Fqy5tMQqwZ+Lz74ccbw48CgYEA5cEhGdNrocPOv3x/IVRN +JLOfXX5nTaiJfxBja1imEIO5ajtoZWjaBdhn2gmqo4+UfyicHfsxrH9RjPX5HmkC +2Yys5gWbcJOr2Wxjd0k+DDFucL+rRsDKxq1vtxov/X0kh/YQ68ydynr0BTbjq04s +1I1KtOPEspYdCKS3+qpcrsECgYBtvYeVesBO9do9G0kMKC26y4bdEwzaz1ASykNn +IrWDHEH6dznr1HqwhHaHsZsvwucWdlmZAAKKWAOkfoU63uYS55qomvPTa9WQwNqS +2koi6Wjh+Al1uvAHvVncKgOwAgar8Nv5ReJBirgPYhSAexpppiRclL/93vNuw7Iq +wvMgkwKBgQC5wnb6SUUrzzKKSRgyusHM/XrjiKgVKq7lvFE9/iJkcw+BEXpjjbEe +RyD0a7PRtCfR39SMVrZp4KXVNNK5ln0WhuLvraMDwOpH9JDWHQiAhuJ3ooSwBylK ++QCLjyOtWAGZAIBRJyb1txfTXZ++dldkOjBi3bmEiadOa48ksvDsNQ== +-----END RSA PRIVATE KEY-----` + validCertificate = `-----BEGIN CERTIFICATE----- +MIIDQjCCAioCCQC6cMRYsE+ahDANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAkxBMQ0wCwYDVQQKDARUZXN0MQ0wCwYD +VQQLDARTdHViMRwwGgYDVQQDDBNob3N0LmNvbnN1bC5leGFtcGxlMB4XDTIzMDIx +NzAyMTA1MloXDTI4MDIxNjAyMTA1MlowYzELMAkGA1UEBhMCVVMxCzAJBgNVBAgM +AkNBMQswCQYDVQQHDAJMQTENMAsGA1UECgwEVGVzdDENMAsGA1UECwwEU3R1YjEc +MBoGA1UEAwwTaG9zdC5jb25zdWwuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANMM2XqJ1JJYTrydAMXHQ3Qk4jMEfbbJv+iBnMPSZudMVWPq +oORhYNUVq0Gjg6uxo6nfyQ1AMLUwZ0DLNcrep27B3tkIWw+F3qrWfun079WaRt5P +BXgfzNNcRaqL3M0Vf4nu+8HeL6qDw/gHHw/DQwBSbd4bcFIBiKpEBz99vUod07S4 +GvWTArNTKE0/jzmBlDy/G8LfUegpTa7nGx+AbOjp19mRWyhe+TvThyjcVk5qQ0qH +bzhWxciqbNwX5z7hdGepo+TOcmXgo5rX0tuT67DN6TARiuWB29DJRX2SKyBm6DG6 +8/RfwTECeuKTHuNysKh7052yloaej8y4RJDxi58CAwEAATANBgkqhkiG9w0BAQsF +AAOCAQEAHF10odRNJ7TKvcD2JPtR8wMacfldSiPcQnn+rhMUyBaKOoSrALxOev+N +L8N+RtEV+KXkyBkvT71OZzEpY9ROwqOQ/acnMdbfG0IBPbg3c/7WDD2sjcdr1zvc +U3T7WJ7G3guZ5aWCuAGgOyT6ZW8nrDa4yFbKZ1PCJkvUQ2ttO1lXmyGPM533Y2pi +SeXP6LL7z5VNqYO3oz5IJEstt10IKxdmb2gKFhHjgEmHN2gFL0jaPi4mjjaINrxq +MdqcM9IzLr26AjZ45NuI9BCcZWO1mraaQTOIb3QL5LyqaC7CRJXLYPSGARthyDhq +J3TrQE3YVrL4D9xnklT86WDnZKApJg== +-----END CERTIFICATE-----` + mismatchedCertificate = `-----BEGIN CERTIFICATE----- +MIIDQjCCAioCCQC2H6+PYz23xDANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAkxBMQ0wCwYDVQQKDARUZXN0MQwwCgYD +VQQLDANGb28xHTAbBgNVBAMMFG90aGVyLmNvbnN1bC5leGFtcGxlMB4XDTIzMDIx +NzAyMTM0OVoXDTI4MDIxNjAyMTM0OVowYzELMAkGA1UEBhMCVVMxCzAJBgNVBAgM +AkNBMQswCQYDVQQHDAJMQTENMAsGA1UECgwEVGVzdDEMMAoGA1UECwwDRm9vMR0w +GwYDVQQDDBRvdGhlci5jb25zdWwuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAO0IH/dzmWJaTPVL32xQVHivrnQk38vskW0ymILYuaismUMJ +0+xrcaTcVljU+3nKhmSW9wcYSFY02GcGWAdcw8x8xO801cna020T+DIWiYaljXT3 +agrbYfULF9q+ihT6IL1D2mFa0AW1x6Bk1XAmZRSTpRBhp7iFNnCXGRK8sSSr95ge +DxaRyj/2F8t6kG+ANPkRBiPd2rRgsYQjuTLuZYBvseeJygnSF8ty1QMg6koz7kdN +bPon3Q5GFH71WNwzm9G3DWjMIu+dhpHz7rsbCnhwLB5lh1jsZBYkAMt3kiyY0g4I +ReuiVWesMe+AMG/DQZvZ5mE252QFJ92dLTeo5RcCAwEAATANBgkqhkiG9w0BAQsF +AAOCAQEAijm6blixjl+pMRAj7EajoPjU+GqhooZayJrvdwvofwcPxQYpkPuh7Uc6 +l2z494b75cRzMw7wS+iW/ad8NYrfw1JwHMsUfncxs5LDO5GsKl9Krg/39goDl3wC +ywTcl00y+FMYfldNPjKDLunENmn+yPa2pKuBVQ0yOKALp+oUeJFVzRNPV5fohlBi +HjypkO0KaVmCG6P01cqCgVkNzxnX9qQYP3YXX1yt5iOcI7QcoOa5WnRhOuD8WqJ1 +v3AZGYNvKyXf9E5nD0y2Cmz6t1awjFjzMlXMx6AdHrjWqxtHhYQ1xz4P4NfzK27m +cCtURSzXMgcrSeZLepBfdICf+0/0+Q== +-----END CERTIFICATE-----` + emptyCNPrivateKey = `-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAx95Opa6t4lGEpiTUogEBptqOdam2ch4BHQGhNhX/MrDwwuZQ httBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2jQlhqTodElkbsd5vWY8R/bxJWQSo NvVE12TlzECxGpJEiHt4W0r8pGffk+rvpljiUyCfnT1kGF3znOSjK1hRMTn6RKWC @@ -31,7 +98,7 @@ T0+9gwKBgHDoerX7NTskg0H0t8O+iSMevdxpEWp34ZYa9gHiftTQGyrRgERCa7Gj nZPAxKb2JoWyfnu3v7G5gZ8fhDFsiOxLbZv6UZJBbUIh1MjJISpXrForDrC2QNLX kHrHfwBFDB3KMudhQknsJzEJKCL/KmFH6o0MvsoaT9yzEl3K+ah/ -----END RSA PRIVATE KEY-----` - validCertificate = `-----BEGIN CERTIFICATE----- + emptyCNCertificate = `-----BEGIN CERTIFICATE----- MIICljCCAX4CCQCQMDsYO8FrPjANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV UzAeFw0yMjEyMjAxNzUwMjVaFw0yNzEyMTkxNzUwMjVaMA0xCzAJBgNVBAYTAlVT MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx95Opa6t4lGEpiTUogEB @@ -46,26 +113,12 @@ RahYIzNLRBTLrwadLAZkApUpZvB8qDK4knsTWFYujNsylCww2A6ajzIMFNU4GkUK NtyHRuD+KYRmjXtyX1yHNqfGN3vOQmwavHq2R8wHYuBSc6LAHHV9vG+j0VsgMELO qwxn8SmLkSKbf2+MsQVzLCXXN5u+D8Yv+4py+oKP4EQ5aFZuDEx+r/G/31rTthww AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r ------END CERTIFICATE-----` - mismatchedCertificate = `-----BEGIN CERTIFICATE----- -MIICljCCAX4CCQC49bq8e0QgLDANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV -UzAeFw0yMjEyMjAxNzUyMzJaFw0yNzEyMTkxNzUyMzJaMA0xCzAJBgNVBAYTAlVT -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk7Are9ulVDY0IqaG5Pt/ -OVuS0kmDhgVUfQBM5JDGRfIsu1ebn68kn5JGCTQ+nC8nU9QXRJS7vG6As5GWm08W -FpkOyIbHLjOhWtYCYzQ+0R+sSSoMnczgl8l6wIUIkR3Vpoy6QUsSZbvo4/xDi3Uk -1CF+JMTM2oFDLD8PNrNzW/txRyTugK36W1G1ofUhvP6EHsTjmVcZwBcLOKToov6L -Ai758MLztl1/X/90DNdZwuHC9fGIgx52Ojz3+XIocXFttr+J8xZglMCtqL4n40bh -5b1DE+hC3NHQmA+7Chc99z28baj2cU1woNk/TO+ewqpyvj+WPWwGOQt3U63ZoPaw -yQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCMF3JlrDdcSv2KYrxEp1tWB/GglI8a -JiSvrf3hePaRz59099bg4DoHzTn0ptOcOPOO9epDPbCJrUqLuPlwvrQRvll6GaW1 -y3TcbnE1AbwTAjbOTgpLhvuj6IVlyNNLoKbjZqs4A8N8i6UkQ7Y8qg77lwxD3QoH -pWLwGZKJifKPa7ObVWmKj727kbU59nA2Hx+Y4qa/MyiPWxJM9Y0JsFGxSBxp4kmQ -q4ikzSWaPv/TvtV+d4mO1H44aggdNMCYIQd/5BXQzG40l+ecHnBueJyG312ax/Zp -NsYUAKQT864cGlxrnWVgT4sW/tsl9Qen7g9iAdeBAPvLO7cQjAjtc7KZ -----END CERTIFICATE-----` ) func TestInlineCertificate(t *testing.T) { + t.Parallel() + cases := map[string]configEntryTestcase{ "invalid private key": { entry: &InlineCertificateConfigEntry{ @@ -101,6 +154,15 @@ func TestInlineCertificate(t *testing.T) { Certificate: validCertificate, }, }, + "empty cn certificate": { + entry: &InlineCertificateConfigEntry{ + Kind: InlineCertificate, + Name: "cert-five", + PrivateKey: emptyCNPrivateKey, + Certificate: emptyCNCertificate, + }, + validateErr: "host \"\" must be a valid DNS hostname", + }, } testConfigEntryNormalizeAndValidate(t, cases) } diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 26e50b3f2d28..027c8f7f7e79 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/hashicorp/consul/acl" + "github.com/miekg/dns" ) // BoundRoute indicates a route that has parent gateways which @@ -41,15 +42,22 @@ type HTTPRouteConfigEntry struct { RaftIndex } -func (e *HTTPRouteConfigEntry) GetServices() []HTTPService { - targets := []HTTPService{} - for _, rule := range e.Rules { - for _, service := range rule.Services { - targets = append(targets, service) - } - } - return targets -} +func (e *HTTPRouteConfigEntry) GetKind() string { return HTTPRoute } +func (e *HTTPRouteConfigEntry) GetName() string { return e.Name } +func (e *HTTPRouteConfigEntry) GetMeta() map[string]string { return e.Meta } +func (e *HTTPRouteConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { return &e.EnterpriseMeta } +func (e *HTTPRouteConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } + +var _ ControlledConfigEntry = (*HTTPRouteConfigEntry)(nil) + +func (e *HTTPRouteConfigEntry) GetStatus() Status { return e.Status } +func (e *HTTPRouteConfigEntry) SetStatus(status Status) { e.Status = status } +func (e *HTTPRouteConfigEntry) DefaultStatus() Status { return Status{} } + +var _ BoundRoute = (*HTTPRouteConfigEntry)(nil) + +func (e *HTTPRouteConfigEntry) GetParents() []ResourceReference { return e.Parents } +func (e *HTTPRouteConfigEntry) GetProtocol() APIGatewayListenerProtocol { return ListenerProtocolHTTP } func (e *HTTPRouteConfigEntry) GetServiceNames() []ServiceName { services := []ServiceName{} @@ -59,67 +67,219 @@ func (e *HTTPRouteConfigEntry) GetServiceNames() []ServiceName { return services } -func (e *HTTPRouteConfigEntry) GetKind() string { - return HTTPRoute +func (e *HTTPRouteConfigEntry) GetServices() []HTTPService { + targets := []HTTPService{} + for _, rule := range e.Rules { + targets = append(targets, rule.Services...) + } + return targets } -func (e *HTTPRouteConfigEntry) GetName() string { - if e == nil { - return "" +func (e *HTTPRouteConfigEntry) Normalize() error { + for i, parent := range e.Parents { + if parent.Kind == "" { + parent.Kind = APIGateway + e.Parents[i] = parent + } + } + + for i, rule := range e.Rules { + for j, match := range rule.Matches { + rule.Matches[j] = normalizeHTTPMatch(match) + } + e.Rules[i] = rule } - return e.Name + + return nil } -func (e *HTTPRouteConfigEntry) GetParents() []ResourceReference { - if e == nil { - return []ResourceReference{} +func normalizeHTTPMatch(match HTTPMatch) HTTPMatch { + method := string(match.Method) + method = strings.ToUpper(method) + match.Method = HTTPMatchMethod(method) + + pathMatch := match.Path.Match + if string(pathMatch) == "" { + match.Path.Match = HTTPPathMatchPrefix + match.Path.Value = "/" } - return e.Parents + + return match } -func (e *HTTPRouteConfigEntry) GetProtocol() APIGatewayListenerProtocol { - return ListenerProtocolHTTP +func (e *HTTPRouteConfigEntry) Validate() error { + for _, host := range e.Hostnames { + // validate that each host referenced in a valid dns name and has + // no wildcards in it + if _, ok := dns.IsDomainName(host); !ok { + return fmt.Errorf("host %q must be a valid DNS hostname", host) + } + + if strings.ContainsRune(host, '*') { + return fmt.Errorf("host %q must not be a wildcard", host) + } + } + + validParentKinds := map[string]bool{ + APIGateway: true, + } + + for _, parent := range e.Parents { + if !validParentKinds[parent.Kind] { + return fmt.Errorf("unsupported parent kind: %q, must be 'api-gateway'", parent.Kind) + } + } + + if err := validateConfigEntryMeta(e.Meta); err != nil { + return err + } + + for i, rule := range e.Rules { + if err := validateRule(rule); err != nil { + return fmt.Errorf("Rule[%d], %w", i, err) + } + } + + return nil } -func (e *HTTPRouteConfigEntry) Normalize() error { +func validateRule(rule HTTPRouteRule) error { + if err := validateFilters(rule.Filters); err != nil { + return err + } + + for i, match := range rule.Matches { + if err := validateMatch(match); err != nil { + return fmt.Errorf("Match[%d], %w", i, err) + } + } + + for i, service := range rule.Services { + if err := validateHTTPService(service); err != nil { + return fmt.Errorf("Service[%d], %w", i, err) + } + } + return nil } -func (e *HTTPRouteConfigEntry) Validate() error { +func validateMatch(match HTTPMatch) error { + if match.Method != HTTPMatchMethodAll { + if !isValidHTTPMethod(string(match.Method)) { + return fmt.Errorf("Method contains an invalid method %q", match.Method) + } + } + + for i, query := range match.Query { + if err := validateHTTPQueryMatch(query); err != nil { + return fmt.Errorf("Query[%d], %w", i, err) + } + } + + for i, header := range match.Headers { + if err := validateHTTPHeaderMatch(header); err != nil { + return fmt.Errorf("Headers[%d], %w", i, err) + } + } + + if err := validateHTTPPathMatch(match.Path); err != nil { + return fmt.Errorf("Path, %w", err) + } + return nil } -func (e *HTTPRouteConfigEntry) CanRead(authz acl.Authorizer) error { - var authzContext acl.AuthorizerContext - e.FillAuthzContext(&authzContext) - return authz.ToAllowAuthorizer().MeshReadAllowed(&authzContext) +func validateHTTPService(service HTTPService) error { + return validateFilters(service.Filters) } -func (e *HTTPRouteConfigEntry) CanWrite(authz acl.Authorizer) error { - var authzContext acl.AuthorizerContext - e.FillAuthzContext(&authzContext) - return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext) +func validateFilters(filter HTTPFilters) error { + for i, header := range filter.Headers { + if err := validateHeaderFilter(header); err != nil { + return fmt.Errorf("HTTPFilters, Headers[%d], %w", i, err) + } + } + + for i, rewrite := range filter.URLRewrites { + if err := validateURLRewrite(rewrite); err != nil { + return fmt.Errorf("HTTPFilters, URLRewrite[%d], %w", i, err) + } + } + + return nil +} + +func validateURLRewrite(rewrite URLRewrite) error { + // TODO: we don't really have validation of the actual params + // passed as "PrefixRewrite" in our discoverychain config + // entries, figure out if we should have something here + return nil +} + +func validateHeaderFilter(filter HTTPHeaderFilter) error { + // TODO: we don't really have validation of the values + // passed as header modifiers in our current discoverychain + // config entries, figure out if we need to + return nil } -func (e *HTTPRouteConfigEntry) GetMeta() map[string]string { - if e == nil { +func validateHTTPQueryMatch(query HTTPQueryMatch) error { + if query.Name == "" { + return fmt.Errorf("missing required Name field") + } + + switch query.Match { + case HTTPQueryMatchExact, + HTTPQueryMatchPresent, + HTTPQueryMatchRegularExpression: return nil + default: + return fmt.Errorf("match type should be one of present, exact, or regex") } - return e.Meta } -func (e *HTTPRouteConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { - if e == nil { +func validateHTTPHeaderMatch(header HTTPHeaderMatch) error { + if header.Name == "" { + return fmt.Errorf("missing required Name field") + } + + switch header.Match { + case HTTPHeaderMatchExact, + HTTPHeaderMatchPrefix, + HTTPHeaderMatchRegularExpression, + HTTPHeaderMatchSuffix, + HTTPHeaderMatchPresent: return nil + default: + return fmt.Errorf("match type should be one of present, exact, prefix, suffix, or regex") } - return &e.EnterpriseMeta } -func (e *HTTPRouteConfigEntry) GetRaftIndex() *RaftIndex { - if e == nil { - return &RaftIndex{} +func validateHTTPPathMatch(path HTTPPathMatch) error { + switch path.Match { + case HTTPPathMatchExact, + HTTPPathMatchPrefix: + if !strings.HasPrefix(path.Value, "/") { + return fmt.Errorf("%s type match doesn't start with '/': %q", path.Match, path.Value) + } + fallthrough + case HTTPPathMatchRegularExpression: + return nil + default: + return fmt.Errorf("match type should be one of exact, prefix, or regex") } - return &e.RaftIndex +} + +func (e *HTTPRouteConfigEntry) CanRead(authz acl.Authorizer) error { + var authzContext acl.AuthorizerContext + e.FillAuthzContext(&authzContext) + return authz.ToAllowAuthorizer().MeshReadAllowed(&authzContext) +} + +func (e *HTTPRouteConfigEntry) CanWrite(authz acl.Authorizer) error { + var authzContext acl.AuthorizerContext + e.FillAuthzContext(&authzContext) + return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext) } func (e *HTTPRouteConfigEntry) FilteredHostnames(listenerHostname string) []string { @@ -279,20 +439,6 @@ func (s HTTPService) ServiceName() ServiceName { return NewServiceName(s.Name, &s.EnterpriseMeta) } -var _ ControlledConfigEntry = (*HTTPRouteConfigEntry)(nil) - -func (e *HTTPRouteConfigEntry) GetStatus() Status { - return e.Status -} - -func (e *HTTPRouteConfigEntry) SetStatus(status Status) { - e.Status = status -} - -func (e *HTTPRouteConfigEntry) DefaultStatus() Status { - return Status{} -} - // TCPRouteConfigEntry manages the configuration for a TCP route // with the given name. type TCPRouteConfigEntry struct { @@ -317,9 +463,22 @@ type TCPRouteConfigEntry struct { RaftIndex } -func (e *TCPRouteConfigEntry) GetServices() []TCPService { - return e.Services -} +func (e *TCPRouteConfigEntry) GetKind() string { return TCPRoute } +func (e *TCPRouteConfigEntry) GetName() string { return e.Name } +func (e *TCPRouteConfigEntry) GetMeta() map[string]string { return e.Meta } +func (e *TCPRouteConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex } +func (e *TCPRouteConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { return &e.EnterpriseMeta } + +var _ ControlledConfigEntry = (*TCPRouteConfigEntry)(nil) + +func (e *TCPRouteConfigEntry) GetStatus() Status { return e.Status } +func (e *TCPRouteConfigEntry) SetStatus(status Status) { e.Status = status } +func (e *TCPRouteConfigEntry) DefaultStatus() Status { return Status{} } + +var _ BoundRoute = (*TCPRouteConfigEntry)(nil) + +func (e *TCPRouteConfigEntry) GetParents() []ResourceReference { return e.Parents } +func (e *TCPRouteConfigEntry) GetProtocol() APIGatewayListenerProtocol { return ListenerProtocolTCP } func (e *TCPRouteConfigEntry) GetServiceNames() []ServiceName { services := []ServiceName{} @@ -329,34 +488,7 @@ func (e *TCPRouteConfigEntry) GetServiceNames() []ServiceName { return services } -func (e *TCPRouteConfigEntry) GetKind() string { - return TCPRoute -} - -func (e *TCPRouteConfigEntry) GetName() string { - if e == nil { - return "" - } - return e.Name -} - -func (e *TCPRouteConfigEntry) GetParents() []ResourceReference { - if e == nil { - return []ResourceReference{} - } - return e.Parents -} - -func (e *TCPRouteConfigEntry) GetProtocol() APIGatewayListenerProtocol { - return ListenerProtocolTCP -} - -func (e *TCPRouteConfigEntry) GetMeta() map[string]string { - if e == nil { - return nil - } - return e.Meta -} +func (e *TCPRouteConfigEntry) GetServices() []TCPService { return e.Services } func (e *TCPRouteConfigEntry) Normalize() error { for i, parent := range e.Parents { @@ -381,6 +513,11 @@ func (e *TCPRouteConfigEntry) Validate() error { return fmt.Errorf("unsupported parent kind: %q, must be 'api-gateway'", parent.Kind) } } + + if err := validateConfigEntryMeta(e.Meta); err != nil { + return err + } + return nil } @@ -396,34 +533,6 @@ func (e *TCPRouteConfigEntry) CanWrite(authz acl.Authorizer) error { return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext) } -func (e *TCPRouteConfigEntry) GetRaftIndex() *RaftIndex { - if e == nil { - return &RaftIndex{} - } - return &e.RaftIndex -} - -func (e *TCPRouteConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta { - if e == nil { - return nil - } - return &e.EnterpriseMeta -} - -var _ ControlledConfigEntry = (*TCPRouteConfigEntry)(nil) - -func (e *TCPRouteConfigEntry) GetStatus() Status { - return e.Status -} - -func (e *TCPRouteConfigEntry) SetStatus(status Status) { - e.Status = status -} - -func (e *TCPRouteConfigEntry) DefaultStatus() Status { - return Status{} -} - // TCPService is a service reference for a TCPRoute type TCPService struct { Name string diff --git a/agent/structs/config_entry_routes_test.go b/agent/structs/config_entry_routes_test.go index dc89ca9e090a..ecacafdbf5a1 100644 --- a/agent/structs/config_entry_routes_test.go +++ b/agent/structs/config_entry_routes_test.go @@ -7,6 +7,8 @@ import ( ) func TestTCPRoute(t *testing.T) { + t.Parallel() + cases := map[string]configEntryTestcase{ "multiple services": { entry: &TCPRouteConfigEntry{ @@ -56,3 +58,207 @@ func TestTCPRoute(t *testing.T) { } testConfigEntryNormalizeAndValidate(t, cases) } + +func TestHTTPRoute(t *testing.T) { + t.Parallel() + + cases := map[string]configEntryTestcase{ + "normalize parent kind": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-one", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + }, + normalizeOnly: true, + check: func(t *testing.T, entry ConfigEntry) { + expectedParent := ResourceReference{ + Kind: APIGateway, + Name: "gateway", + } + route := entry.(*HTTPRouteConfigEntry) + require.Len(t, route.Parents, 1) + require.Equal(t, expectedParent, route.Parents[0]) + }, + }, + "invalid parent kind": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Kind: "route", + Name: "gateway", + }}, + }, + validateErr: "unsupported parent kind", + }, + "wildcard hostnames": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Hostnames: []string{"*"}, + }, + validateErr: "host \"*\" must not be a wildcard", + }, + "wildcard subdomain": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Hostnames: []string{"*.consul.example"}, + }, + validateErr: "host \"*.consul.example\" must not be a wildcard", + }, + "valid dns hostname": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Hostnames: []string{"...not legal"}, + }, + validateErr: "host \"...not legal\" must be a valid DNS hostname", + }, + "rule matches invalid header match type": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Headers: []HTTPHeaderMatch{{ + Match: HTTPHeaderMatchType("foo"), + Name: "foo", + }}, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Headers[0], match type should be one of present, exact, prefix, suffix, or regex", + }, + "rule matches invalid header match name": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Headers: []HTTPHeaderMatch{{ + Match: HTTPHeaderMatchPresent, + }}, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Headers[0], missing required Name field", + }, + "rule matches invalid query match type": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Query: []HTTPQueryMatch{{ + Match: HTTPQueryMatchType("foo"), + Name: "foo", + }}, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Query[0], match type should be one of present, exact, or regex", + }, + "rule matches invalid query match name": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Query: []HTTPQueryMatch{{ + Match: HTTPQueryMatchPresent, + }}, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Query[0], missing required Name field", + }, + "rule matches invalid path match type": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Path: HTTPPathMatch{ + Match: HTTPPathMatchType("foo"), + }, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Path, match type should be one of exact, prefix, or regex", + }, + "rule matches invalid path match prefix": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Path: HTTPPathMatch{ + Match: HTTPPathMatchPrefix, + }, + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Path, prefix type match doesn't start with '/': \"\"", + }, + "rule matches invalid method": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Method: HTTPMatchMethod("foo"), + }}, + }}, + }, + validateErr: "Rule[0], Match[0], Method contains an invalid method \"FOO\"", + }, + "rule normalizes method casing and path matches": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{{ + Name: "gateway", + }}, + Rules: []HTTPRouteRule{{ + Matches: []HTTPMatch{{ + Method: HTTPMatchMethod("trace"), + }}, + }}, + }, + }, + } + testConfigEntryNormalizeAndValidate(t, cases) +} diff --git a/api/config_entry_inline_certificate_test.go b/api/config_entry_inline_certificate_test.go index 3b1cc1f54ef6..78771fcc78cf 100644 --- a/api/config_entry_inline_certificate_test.go +++ b/api/config_entry_inline_certificate_test.go @@ -10,47 +10,51 @@ import ( const ( // generated via openssl req -x509 -sha256 -days 1825 -newkey rsa:2048 -keyout private.key -out certificate.crt validPrivateKey = `-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAx95Opa6t4lGEpiTUogEBptqOdam2ch4BHQGhNhX/MrDwwuZQ -httBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2jQlhqTodElkbsd5vWY8R/bxJWQSo -NvVE12TlzECxGpJEiHt4W0r8pGffk+rvpljiUyCfnT1kGF3znOSjK1hRMTn6RKWC -yYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409g9X5VU88/Bmmrz4cMyxce86Kc2ug -5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftrXOvuCbO5IBRHMOBHiHTZ4rtGuhMa -Ir21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+WmQIDAQABAoIBACYvceUzp2MK4gYA -GWPOP2uKbBdM0l+hHeNV0WAM+dHMfmMuL4pkT36ucqt0ySOLjw6rQyOZG5nmA6t9 -sv0g4ae2eCMlyDIeNi1Yavu4Wt6YX4cTXbQKThm83C6W2X9THKbauBbxD621bsDK -7PhiGPN60yPue7YwFQAPqqD4YaK+s22HFIzk9gwM/rkvAUNwRv7SyHMiFe4Igc1C -Eev7iHWzvj5Heoz6XfF+XNF9DU+TieSUAdjd56VyUb8XL4+uBTOhHwLiXvAmfaMR -HvpcxeKnYZusS6NaOxcUHiJnsLNWrxmJj9WEGgQzuLxcLjTe4vVmELVZD8t3QUKj -PAxu8tUCgYEA7KIWVn9dfVpokReorFym+J8FzLwSktP9RZYEMonJo00i8aii3K9s -u/aSwRWQSCzmON1ZcxZzWhwQF9usz6kGCk//9+4hlVW90GtNK0RD+j7sp4aT2JI8 -9eLEjTG+xSXa7XWe98QncjjL9lu/yrRncSTxHs13q/XP198nn2aYuQ8CgYEA2Dnt -sRBzv0fFEvzzFv7G/5f85mouN38TUYvxNRTjBLCXl9DeKjDkOVZ2b6qlfQnYXIru -H+W+v+AZEb6fySXc8FRab7lkgTMrwE+aeI4rkW7asVwtclv01QJ5wMnyT84AgDD/ -Dgt/RThFaHgtU9TW5GOZveL+l9fVPn7vKFdTJdcCgYEArJ99zjHxwJ1whNAOk1av -09UmRPm6TvRo4heTDk8oEoIWCNatoHI0z1YMLuENNSnT9Q280FFDayvnrY/qnD7A -kktT/sjwJOG8q8trKzIMqQS4XWm2dxoPcIyyOBJfCbEY6XuRsUuePxwh5qF942EB -yS9a2s6nC4Ix0lgPrqAIr48CgYBgS/Q6riwOXSU8nqCYdiEkBYlhCJrKpnJxF9T1 -ofa0yPzKZP/8ZEfP7VzTwHjxJehQ1qLUW9pG08P2biH1UEKEWdzo8vT6wVJT1F/k -HtTycR8+a+Hlk2SHVRHqNUYQGpuIe8mrdJ1as4Pd0d/F/P0zO9Rlh+mAsGPM8HUM -T0+9gwKBgHDoerX7NTskg0H0t8O+iSMevdxpEWp34ZYa9gHiftTQGyrRgERCa7Gj -nZPAxKb2JoWyfnu3v7G5gZ8fhDFsiOxLbZv6UZJBbUIh1MjJISpXrForDrC2QNLX -kHrHfwBFDB3KMudhQknsJzEJKCL/KmFH6o0MvsoaT9yzEl3K+ah/ +MIIEpAIBAAKCAQEA0wzZeonUklhOvJ0AxcdDdCTiMwR9tsm/6IGcw9Jm50xVY+qg +5GFg1RWrQaODq7Gjqd/JDUAwtTBnQMs1yt6nbsHe2QhbD4XeqtZ+6fTv1ZpG3k8F +eB/M01xFqovczRV/ie77wd4vqoPD+AcfD8NDAFJt3htwUgGIqkQHP329Sh3TtLga +9ZMCs1MoTT+POYGUPL8bwt9R6ClNrucbH4Bs6OnX2ZFbKF75O9OHKNxWTmpDSodv +OFbFyKps3BfnPuF0Z6mj5M5yZeCjmtfS25PrsM3pMBGK5YHb0MlFfZIrIGboMbrz +9F/BMQJ64pMe43KwqHvTnbKWhp6PzLhEkPGLnwIDAQABAoIBADBEJAiONPszDu67 +yU1yAM8zEDgysr127liyK7PtDnOfVXgAVMNmMcsJpZzhVF+TxKY487YAFCOb6kE7 +OBYpTYla9SgVbR3js8TGQUgoKCFlowd8cvfB7gn4dEZIrjqIzB4zdYgk1Cne8JZs +qoHkWhJcx5ugEtPuXd7yp+WxT/T+6uOro06scp67NhP5t9yoAGFv5Vdb577RuzRo +Wkd9higQ9A20+GtjCY0EYxdgRviWvW7mM5/F+Lzcaui86ME+ga754gX8zgW3+NJ5 +LMsz5OLSnh291Uyjmr77HWBv/xvpq01Fls0LyJcgxFVZuJs5GQz+l3otSqv4FTP6 +Ua9w/YECgYEA8To3dgUK1QhzX5rwhWtlst3pItGTvmEdNzXmjgSylu7uKM13i+xg +llhp2uXrOEtuL+xtBZdeFNaijusbyqjg0xj6e4o31c19okuuDkJD5/sfQq22bvrn +gVJMGuESprIiPePrEyrXCHOdxH6eDgR2dIzAeO5vz0nnKGFAWrJJbvECgYEA3/mJ +eacXOJznw4Sa8jGWS2FtZLKxDHph7uDKMJmuG0ukb3aHJ9dMHrPleCLo8mhpoObA +hueoIbIP7swGrQx79+nZbnQpF6rMp6FAU5bF3gSrj1eWbaeh8pn9mrv4hal9USmn +orTbXMxDp3XSh7voR8Fqy5tMQqwZ+Lz74ccbw48CgYEA5cEhGdNrocPOv3x/IVRN +JLOfXX5nTaiJfxBja1imEIO5ajtoZWjaBdhn2gmqo4+UfyicHfsxrH9RjPX5HmkC +2Yys5gWbcJOr2Wxjd0k+DDFucL+rRsDKxq1vtxov/X0kh/YQ68ydynr0BTbjq04s +1I1KtOPEspYdCKS3+qpcrsECgYBtvYeVesBO9do9G0kMKC26y4bdEwzaz1ASykNn +IrWDHEH6dznr1HqwhHaHsZsvwucWdlmZAAKKWAOkfoU63uYS55qomvPTa9WQwNqS +2koi6Wjh+Al1uvAHvVncKgOwAgar8Nv5ReJBirgPYhSAexpppiRclL/93vNuw7Iq +wvMgkwKBgQC5wnb6SUUrzzKKSRgyusHM/XrjiKgVKq7lvFE9/iJkcw+BEXpjjbEe +RyD0a7PRtCfR39SMVrZp4KXVNNK5ln0WhuLvraMDwOpH9JDWHQiAhuJ3ooSwBylK ++QCLjyOtWAGZAIBRJyb1txfTXZ++dldkOjBi3bmEiadOa48ksvDsNQ== -----END RSA PRIVATE KEY-----` validCertificate = `-----BEGIN CERTIFICATE----- -MIICljCCAX4CCQCQMDsYO8FrPjANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV -UzAeFw0yMjEyMjAxNzUwMjVaFw0yNzEyMTkxNzUwMjVaMA0xCzAJBgNVBAYTAlVT -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx95Opa6t4lGEpiTUogEB -ptqOdam2ch4BHQGhNhX/MrDwwuZQhttBwMfngQ/wd9NmYEPAwj0dumUoAITIq6i2 -jQlhqTodElkbsd5vWY8R/bxJWQSoNvVE12TlzECxGpJEiHt4W0r8pGffk+rvplji -UyCfnT1kGF3znOSjK1hRMTn6RKWCyYaBvXQiB4SGilfLgJcEpOJKtISIxmZ+S409 -g9X5VU88/Bmmrz4cMyxce86Kc2ug5/MOv0CjWDJwlrv8njneV2zvraQ61DDwQftr -XOvuCbO5IBRHMOBHiHTZ4rtGuhMaIr21V4vb6n8c4YzXiFvhUYcyX7rltGZzVd+W -mQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBfCqoUIdPf/HGSbOorPyZWbyizNtHJ -GL7x9cAeIYxpI5Y/WcO1o5v94lvrgm3FNfJoGKbV66+JxOge731FrfMpHplhar1Z -RahYIzNLRBTLrwadLAZkApUpZvB8qDK4knsTWFYujNsylCww2A6ajzIMFNU4GkUK -NtyHRuD+KYRmjXtyX1yHNqfGN3vOQmwavHq2R8wHYuBSc6LAHHV9vG+j0VsgMELO -qwxn8SmLkSKbf2+MsQVzLCXXN5u+D8Yv+4py+oKP4EQ5aFZuDEx+r/G/31rTthww -AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r +MIIDQjCCAioCCQC6cMRYsE+ahDANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAkxBMQ0wCwYDVQQKDARUZXN0MQ0wCwYD +VQQLDARTdHViMRwwGgYDVQQDDBNob3N0LmNvbnN1bC5leGFtcGxlMB4XDTIzMDIx +NzAyMTA1MloXDTI4MDIxNjAyMTA1MlowYzELMAkGA1UEBhMCVVMxCzAJBgNVBAgM +AkNBMQswCQYDVQQHDAJMQTENMAsGA1UECgwEVGVzdDENMAsGA1UECwwEU3R1YjEc +MBoGA1UEAwwTaG9zdC5jb25zdWwuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANMM2XqJ1JJYTrydAMXHQ3Qk4jMEfbbJv+iBnMPSZudMVWPq +oORhYNUVq0Gjg6uxo6nfyQ1AMLUwZ0DLNcrep27B3tkIWw+F3qrWfun079WaRt5P +BXgfzNNcRaqL3M0Vf4nu+8HeL6qDw/gHHw/DQwBSbd4bcFIBiKpEBz99vUod07S4 +GvWTArNTKE0/jzmBlDy/G8LfUegpTa7nGx+AbOjp19mRWyhe+TvThyjcVk5qQ0qH +bzhWxciqbNwX5z7hdGepo+TOcmXgo5rX0tuT67DN6TARiuWB29DJRX2SKyBm6DG6 +8/RfwTECeuKTHuNysKh7052yloaej8y4RJDxi58CAwEAATANBgkqhkiG9w0BAQsF +AAOCAQEAHF10odRNJ7TKvcD2JPtR8wMacfldSiPcQnn+rhMUyBaKOoSrALxOev+N +L8N+RtEV+KXkyBkvT71OZzEpY9ROwqOQ/acnMdbfG0IBPbg3c/7WDD2sjcdr1zvc +U3T7WJ7G3guZ5aWCuAGgOyT6ZW8nrDa4yFbKZ1PCJkvUQ2ttO1lXmyGPM533Y2pi +SeXP6LL7z5VNqYO3oz5IJEstt10IKxdmb2gKFhHjgEmHN2gFL0jaPi4mjjaINrxq +MdqcM9IzLr26AjZ45NuI9BCcZWO1mraaQTOIb3QL5LyqaC7CRJXLYPSGARthyDhq +J3TrQE3YVrL4D9xnklT86WDnZKApJg== -----END CERTIFICATE-----` ) From 5437e5b7891256ec483a7768c7306fb09deb92ee Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Feb 2023 17:53:13 -0500 Subject: [PATCH 012/421] backport of commit d3daf89ba88501322d823e2d1e0b29c4b04f4d52 (#16324) Co-authored-by: Andrew Stucki --- agent/structs/config_entry_routes_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/agent/structs/config_entry_routes_test.go b/agent/structs/config_entry_routes_test.go index ecacafdbf5a1..83ab8c4d79e0 100644 --- a/agent/structs/config_entry_routes_test.go +++ b/agent/structs/config_entry_routes_test.go @@ -3,6 +3,7 @@ package structs import ( "testing" + "github.com/hashicorp/consul/acl" "github.com/stretchr/testify/require" ) @@ -36,8 +37,9 @@ func TestTCPRoute(t *testing.T) { normalizeOnly: true, check: func(t *testing.T, entry ConfigEntry) { expectedParent := ResourceReference{ - Kind: APIGateway, - Name: "gateway", + Kind: APIGateway, + Name: "gateway", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), } route := entry.(*TCPRouteConfigEntry) require.Len(t, route.Parents, 1) @@ -74,8 +76,9 @@ func TestHTTPRoute(t *testing.T) { normalizeOnly: true, check: func(t *testing.T, entry ConfigEntry) { expectedParent := ResourceReference{ - Kind: APIGateway, - Name: "gateway", + Kind: APIGateway, + Name: "gateway", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), } route := entry.(*HTTPRouteConfigEntry) require.Len(t, route.Parents, 1) From 065ab57133b6f7d7e2d818034a7f23e60697cf36 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Feb 2023 18:23:16 -0500 Subject: [PATCH 013/421] backport of commit acf20db763af726d9da4f89055ed136a6533b829 (#16320) Co-authored-by: Andrew Stucki --- agent/structs/config_entry_gateways.go | 24 ++++++++++++++++++++---- agent/structs/config_entry_routes.go | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 0d4019896f3a..c757dce984f2 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -754,6 +754,8 @@ func (e *APIGatewayConfigEntry) Normalize() error { if cert.Kind == "" { cert.Kind = InlineCertificate } + cert.EnterpriseMeta.Normalize() + listener.TLS.Certificates[i] = cert } } @@ -972,7 +974,23 @@ func (e *BoundAPIGatewayConfigEntry) IsInitializedForGateway(gateway *APIGateway func (e *BoundAPIGatewayConfigEntry) GetKind() string { return BoundAPIGateway } func (e *BoundAPIGatewayConfigEntry) GetName() string { return e.Name } func (e *BoundAPIGatewayConfigEntry) GetMeta() map[string]string { return e.Meta } -func (e *BoundAPIGatewayConfigEntry) Normalize() error { return nil } +func (e *BoundAPIGatewayConfigEntry) Normalize() error { + for i, listener := range e.Listeners { + for j, route := range listener.Routes { + route.EnterpriseMeta.Normalize() + + listener.Routes[j] = route + } + for j, cert := range listener.Certificates { + cert.EnterpriseMeta.Normalize() + + listener.Certificates[j] = cert + } + + e.Listeners[i] = listener + } + return nil +} func (e *BoundAPIGatewayConfigEntry) Validate() error { allowedCertificateKinds := map[string]bool{ @@ -1109,8 +1127,6 @@ func (l *BoundAPIGatewayListener) UnbindRoute(route ResourceReference) bool { return false } -func (e *BoundAPIGatewayConfigEntry) GetStatus() Status { - return Status{} -} +func (e *BoundAPIGatewayConfigEntry) GetStatus() Status { return Status{} } func (e *BoundAPIGatewayConfigEntry) SetStatus(status Status) {} func (e *BoundAPIGatewayConfigEntry) DefaultStatus() Status { return Status{} } diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 027c8f7f7e79..fa4427776a40 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -79,6 +79,7 @@ func (e *HTTPRouteConfigEntry) Normalize() error { for i, parent := range e.Parents { if parent.Kind == "" { parent.Kind = APIGateway + parent.EnterpriseMeta.Normalize() e.Parents[i] = parent } } @@ -87,12 +88,22 @@ func (e *HTTPRouteConfigEntry) Normalize() error { for j, match := range rule.Matches { rule.Matches[j] = normalizeHTTPMatch(match) } + + for j, service := range rule.Services { + rule.Services[j] = normalizeHTTPService(service) + } e.Rules[i] = rule } return nil } +func normalizeHTTPService(service HTTPService) HTTPService { + service.EnterpriseMeta.Normalize() + + return service +} + func normalizeHTTPMatch(match HTTPMatch) HTTPMatch { method := string(match.Method) method = strings.ToUpper(method) @@ -494,9 +505,16 @@ func (e *TCPRouteConfigEntry) Normalize() error { for i, parent := range e.Parents { if parent.Kind == "" { parent.Kind = APIGateway + parent.EnterpriseMeta.Normalize() e.Parents[i] = parent } } + + for i, service := range e.Services { + service.EnterpriseMeta.Normalize() + e.Services[i] = service + } + return nil } From a3a9b05e141ebcc7ce6a89654800103a500c8e00 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Sat, 18 Feb 2023 15:17:58 -0500 Subject: [PATCH 014/421] backport of commit 748e2a76cbf87eeca30a46561b114d85ca1dac66 (#16328) Co-authored-by: DanStough --- api/go.mod | 2 +- envoyextensions/go.mod | 2 +- sdk/freeport/freeport.go | 6 +++++- sdk/go.mod | 2 +- troubleshoot/go.mod | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/api/go.mod b/api/go.mod index d9a68569df7f..c987fde1abef 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/api -go 1.20 +go 1.19 replace github.com/hashicorp/consul/sdk => ../sdk diff --git a/envoyextensions/go.mod b/envoyextensions/go.mod index 5a73e969c2db..f96560804c9d 100644 --- a/envoyextensions/go.mod +++ b/envoyextensions/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/envoyextensions -go 1.20 +go 1.19 replace github.com/hashicorp/consul/api => ../api diff --git a/sdk/freeport/freeport.go b/sdk/freeport/freeport.go index 6c275fe86674..c51ef9815395 100644 --- a/sdk/freeport/freeport.go +++ b/sdk/freeport/freeport.go @@ -76,6 +76,9 @@ var ( // total is the total number of available ports in the block for use. total int + // seededRand is a random generator that is pre-seeded from the current time. + seededRand *rand.Rand + // stopCh is used to signal to background goroutines to terminate. Only // really exists for the safety of reset() during unit tests. stopCh chan struct{} @@ -114,6 +117,7 @@ func initialize() { panic("freeport: block size too big or too many blocks requested") } + seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) // This is compatible with go 1.19 but unnecessary in >= go1.20 firstPort, lockLn = alloc() condNotEmpty = sync.NewCond(&mu) @@ -255,7 +259,7 @@ func adjustMaxBlocks() (int, error) { // be automatically released when the application terminates. func alloc() (int, net.Listener) { for i := 0; i < attempts; i++ { - block := int(rand.Int31n(int32(effectiveMaxBlocks))) + block := int(seededRand.Int31n(int32(effectiveMaxBlocks))) firstPort := lowPort + block*blockSize ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", firstPort)) if err != nil { diff --git a/sdk/go.mod b/sdk/go.mod index 63ad3671a3e1..8a04b9e10350 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/sdk -go 1.20 +go 1.19 require ( github.com/hashicorp/go-cleanhttp v0.5.1 diff --git a/troubleshoot/go.mod b/troubleshoot/go.mod index cf82c88f1e1d..c4e445ee05f1 100644 --- a/troubleshoot/go.mod +++ b/troubleshoot/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul/troubleshoot -go 1.20 +go 1.19 replace github.com/hashicorp/consul/api => ../api From 84bc9713342e80243faa2cc75c0451075bb73235 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 08:43:27 -0500 Subject: [PATCH 015/421] Backport of fix: add tls config to unix socket when https is used into release/1.15.x (#16336) * backport of commit 49f7423ab858f6fe237833a2d42cb5c323f77e10 * backport of commit d5408c8d019d50cf899b8d0a65bd6dd752b40498 --------- Co-authored-by: cskh --- .changelog/16301.txt | 3 ++ agent/agent.go | 3 +- agent/http_test.go | 100 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 .changelog/16301.txt diff --git a/.changelog/16301.txt b/.changelog/16301.txt new file mode 100644 index 000000000000..e1dc5deb1c5b --- /dev/null +++ b/.changelog/16301.txt @@ -0,0 +1,3 @@ +```release-note:bug +agent configuration: Fix issue of using unix socket when https is used. +``` diff --git a/agent/agent.go b/agent/agent.go index bff47a0fcf2d..bd8c701ca794 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1051,7 +1051,8 @@ func (a *Agent) listenHTTP() ([]apiServer, error) { for _, l := range listeners { var tlscfg *tls.Config _, isTCP := l.(*tcpKeepAliveListener) - if isTCP && proto == "https" { + isUnix := l.Addr().Network() == "unix" + if (isTCP || isUnix) && proto == "https" { tlscfg = a.tlsConfigurator.IncomingHTTPSConfig() l = tls.NewListener(l, tlscfg) } diff --git a/agent/http_test.go b/agent/http_test.go index 39963be0417a..7f95e38ce76b 100644 --- a/agent/http_test.go +++ b/agent/http_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "net/netip" + "net/url" "os" "path/filepath" "runtime" @@ -140,6 +141,95 @@ func TestHTTPServer_UnixSocket_FileExists(t *testing.T) { } } +func TestHTTPSServer_UnixSocket(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + if runtime.GOOS == "windows" { + t.SkipNow() + } + + tempDir := testutil.TempDir(t, "consul") + socket := filepath.Join(tempDir, "test.sock") + + a := StartTestAgent(t, TestAgent{ + UseHTTPS: true, + HCL: ` + addresses { + https = "unix://` + socket + `" + } + unix_sockets { + mode = "0777" + } + tls { + defaults { + ca_file = "../test/client_certs/rootca.crt" + cert_file = "../test/client_certs/server.crt" + key_file = "../test/client_certs/server.key" + } + } + `, + }) + defer a.Shutdown() + + // Ensure the socket was created + if _, err := os.Stat(socket); err != nil { + t.Fatalf("err: %s", err) + } + + // Ensure the mode was set properly + fi, err := os.Stat(socket) + if err != nil { + t.Fatalf("err: %s", err) + } + if fi.Mode().String() != "Srwxrwxrwx" { + t.Fatalf("bad permissions: %s", fi.Mode()) + } + + // Make an HTTP/2-enabled client, using the API helpers to set + // up TLS to be as normal as possible for Consul. + tlscfg := &api.TLSConfig{ + Address: "consul.test", + KeyFile: "../test/client_certs/client.key", + CertFile: "../test/client_certs/client.crt", + CAFile: "../test/client_certs/rootca.crt", + } + tlsccfg, err := api.SetupTLSConfig(tlscfg) + if err != nil { + t.Fatalf("err: %v", err) + } + transport := api.DefaultConfig().Transport + transport.TLSHandshakeTimeout = 30 * time.Second + transport.TLSClientConfig = tlsccfg + if err := http2.ConfigureTransport(transport); err != nil { + t.Fatalf("err: %v", err) + } + transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", socket) + } + client := &http.Client{Transport: transport} + + u, err := url.Parse("https://unix" + socket) + if err != nil { + t.Fatalf("err: %s", err) + } + u.Path = "/v1/agent/self" + u.Scheme = "https" + resp, err := client.Get(u.String()) + if err != nil { + t.Fatalf("err: %s", err) + } + defer resp.Body.Close() + + if body, err := io.ReadAll(resp.Body); err != nil || len(body) == 0 { + t.Fatalf("bad: %s %v", body, err) + } else if !strings.Contains(string(body), "NodeName") { + t.Fatalf("NodeName not found in results: %s", string(body)) + } +} + func TestSetupHTTPServer_HTTP2(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") @@ -151,9 +241,13 @@ func TestSetupHTTPServer_HTTP2(t *testing.T) { a := StartTestAgent(t, TestAgent{ UseHTTPS: true, HCL: ` - key_file = "../test/client_certs/server.key" - cert_file = "../test/client_certs/server.crt" - ca_file = "../test/client_certs/rootca.crt" + tls { + defaults { + ca_file = "../test/client_certs/rootca.crt" + cert_file = "../test/client_certs/server.crt" + key_file = "../test/client_certs/server.key" + } + } `, }) defer a.Shutdown() From ca02bf6c73427d34bb929e61106f0d3b1418f30f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 11:36:40 -0500 Subject: [PATCH 016/421] Backport of new docs for consul and consul-k8s troubleshoot command into release/1.15.x (#16318) * backport of commit ba5560c2a2bf57e97557ec74d623d82f9efb05ff * backport of commit 0f3a7cb26b4ac5438c27c21d8d3c82ee4f8420b5 * backport of commit effdb9bb57ea752139ea4371a4d0c877c0e65afa * fix PR from original OSS PR: 16284 * fix missing tab in docs * fix missing tab in docs --------- Co-authored-by: Maliz --- .changelog/16284.txt | 3 + website/content/commands/index.mdx | 1 + .../content/commands/troubleshoot/index.mdx | 31 ++++++ .../content/commands/troubleshoot/proxy.mdx | 65 +++++++++++ .../commands/troubleshoot/upstreams.mdx | 35 ++++++ website/content/docs/k8s/k8s-cli.mdx | 101 ++++++++++++++++++ website/data/commands-nav-data.json | 17 +++ 7 files changed, 253 insertions(+) create mode 100644 .changelog/16284.txt create mode 100644 website/content/commands/troubleshoot/index.mdx create mode 100644 website/content/commands/troubleshoot/proxy.mdx create mode 100644 website/content/commands/troubleshoot/upstreams.mdx diff --git a/.changelog/16284.txt b/.changelog/16284.txt new file mode 100644 index 000000000000..23dd2aa6fef9 --- /dev/null +++ b/.changelog/16284.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: adds new CLI commands `consul troubleshoot upstreams` and `consul troubleshoot proxy` to troubleshoot Consul's service mesh configuration and network issues. +``` \ No newline at end of file diff --git a/website/content/commands/index.mdx b/website/content/commands/index.mdx index 2946d794ba46..415fc551d331 100644 --- a/website/content/commands/index.mdx +++ b/website/content/commands/index.mdx @@ -56,6 +56,7 @@ Available commands are: services Interact with services snapshot Saves, restores and inspects snapshots of Consul server state tls Builtin helpers for creating CAs and certificates + troubleshoot Provides tools to troubleshoot Consul's service mesh configuration validate Validate config files/directories version Prints the Consul version watch Watch for changes in Consul diff --git a/website/content/commands/troubleshoot/index.mdx b/website/content/commands/troubleshoot/index.mdx new file mode 100644 index 000000000000..521981a77e0b --- /dev/null +++ b/website/content/commands/troubleshoot/index.mdx @@ -0,0 +1,31 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot' +description: >- + The `consul troubleshoot` command provides tools to troubleshoot Consul's service mesh configuration. +--- + +# Consul Troubleshooting + +Command: `consul troubleshoot` + +Use the `troubleshoot` command to diagnose Consul service mesh configuration or network issues. + +## Usage + +```text +Usage: consul troubleshoot [options] + + # ... + +Subcommands: + + proxy Troubleshoots service mesh issues from the current Envoy instance + upstreams Gets upstream Envoy identifiers and IPs configured for the proxy +``` + +For more information, examples, and usage about a subcommand, click on the name +of the subcommand in the sidebar or one of the links below: + +- [proxy](/consul/commands/troubleshoot/proxy) +- [upstreams](/consul/commands/troubleshoot/upstreams) diff --git a/website/content/commands/troubleshoot/proxy.mdx b/website/content/commands/troubleshoot/proxy.mdx new file mode 100644 index 000000000000..6c93581b155e --- /dev/null +++ b/website/content/commands/troubleshoot/proxy.mdx @@ -0,0 +1,65 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot Proxy' +description: >- + The `consul troubleshoot proxy` diagnoses Consul service mesh configuration and network issues to an upstream. +--- + +# Consul Troubleshoot Proxy + +Command: `consul troubleshoot proxy` + +The `troubleshoot proxy` command diagnoses Consul service mesh configuration and network issues to an upstream. + +## Usage + +Usage: `consul troubleshoot proxy (-upstream-ip |-upstream-envoy-id ) [options]` +This command requires `-envoy-admin-endpoint` or `-upstream-ip` to specify the upstream. + +#### Command Options + +- `-envoy-admin-endpoint=` - Envoy admin endpoint address for the local Envoy instance. +Defaults to `127.0.0.1:19000`. + +- `-upstream-ip=` - The IP address of the upstream service; Use when the upstream is reached via a transparent proxy DNS address (KubeDNS or Consul DNS) + +- `-upstream-envoy-id=` - The Envoy identifier of the upstream service; Use when the upstream is explicitly configured on the downstream service. + +## Examples + +The following example illustrates how to troubleshoot Consul service mesh configuration and network issues to an upstream from a source proxy. + + ```shell-session + $ consul troubleshoot proxy -upstream-ip 10.4.6.160 + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "backend" found + ✓ Route for upstream "backend" found + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + -> Check that your upstream service is healthy and running + -> Check that your upstream service is registered with Consul + -> Check that the upstream proxy is healthy and running + -> If you are explicitly configuring upstreams, ensure the name of the upstream is correct + ``` + +The following example troubleshoots Consul service mesh configuration and network issues from the local Envoy instance to the given upstream. + + ```shell-session + $ consul-k8s troubleshoot proxy -upstream-envoy-id db + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "db" found + ✓ Route for upstream "db" found + ✓ Cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0.consul" for upstream "db" found + ✓ Healthy endpoints for cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0.consul" for upstream "db" found + If you are still experiencing issues, you can: + -> Check intentions to ensure the upstream allows traffic from this source + -> If using transparent proxy, ensure DNS resolution is to the same IP you have verified here + ``` diff --git a/website/content/commands/troubleshoot/upstreams.mdx b/website/content/commands/troubleshoot/upstreams.mdx new file mode 100644 index 000000000000..752bb0463c51 --- /dev/null +++ b/website/content/commands/troubleshoot/upstreams.mdx @@ -0,0 +1,35 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot Upstreams' +description: >- + The `consul troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. +--- + +# Consul Troubleshoot Upstreams + +Command: `consul troubleshoot upstreams` + +The `troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. + +## Usage + +Usage: `consul troubleshoot upstreams [options]` + +#### Command Options + +- `-envoy-admin-endpoint=` - Envoy admin endpoint address for the local Envoy instance. +Defaults to `127.0.0.1:19000`. + +## Examples + +Display all transparent proxy upstreams in Consul service mesh from the current Envoy instance. + + ```shell-session + $ consul troubleshoot upstreams + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` diff --git a/website/content/docs/k8s/k8s-cli.mdx b/website/content/docs/k8s/k8s-cli.mdx index 402d2327b7d3..b3e6f76bc4e7 100644 --- a/website/content/docs/k8s/k8s-cli.mdx +++ b/website/content/docs/k8s/k8s-cli.mdx @@ -33,6 +33,7 @@ You can use the following commands with `consul-k8s`. - [`proxy list`](#proxy-list): List all Pods running proxies managed by Consul. - [`proxy read`](#proxy-read): Inspect the Envoy configuration for a given Pod. - [`status`](#status): Check the status of a Consul installation on Kubernetes. + - [`troubleshoot`](#troubleshoot): Troubleshoot Consul service mesh and networking issues from a given pod. - [`uninstall`](#uninstall): Uninstall Consul deployment. - [`upgrade`](#upgrade): Upgrade Consul on Kubernetes from an existing installation. - [`version`](#version): Print the version of the Consul on Kubernetes CLI. @@ -490,6 +491,106 @@ $ consul-k8s status ✓ Consul clients healthy (3/3) ``` +### `troubleshoot` + +The `troubleshoot` command exposes two subcommands for troubleshooting Consul +service mesh and network issues from a given pod. + +- [`troubleshoot upstreams`](#troubleshoot-upstreams): List all Envoy upstreams in Consul service mesh from the given pod. +- [`troubleshoot proxy`](#troubleshoot-proxy): Troubleshoot Consul service mesh configuration and network issues between the given pod and the given upstream. + +### `troubleshoot upstreams` + +```shell-session +$ consul-k8s troubleshoot upstreams -pod +``` + +| Flag | Description | Default | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | +| `-envoy-admin-endpoint` | `String` Envoy sidecar address and port | `127.0.0.1:19000` | + +#### Example Commands + +The following example displays all transparent proxy upstreams in Consul service mesh from the given pod. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod frontend-767ccfc8f9-6f6gx + + ==> Upstreams (explicit upstreams only) (0) + + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +The following example displays all explicit upstreams from the given pod in the Consul service mesh. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod client-767ccfc8f9-6f6gx + + ==> Upstreams (explicit upstreams only) (1) + server + counting + + ==> Upstreams IPs (transparent proxy only) (0) + + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +### `troubleshoot proxy` + +```shell-session +$ consul-k8s troubleshoot proxy -pod -upstream-ip +$ consul-k8s troubleshoot proxy -pod -upstream-envoy-id +``` + +| Flag | Description | Default | +| ------------------------------------ | ----------------------------------------------------------| ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | +| `-envoy-admin-endpoint` | `String` Envoy sidecar address and port | `127.0.0.1:19000` | +| `-upstream-ip` | `String` The IP address of the upstream transparent proxy | | +| `-upstream-envoy-id` | `String` The Envoy identifier of the upstream | | + +#### Example Commands + +The following example troubleshoots the Consul service mesh configuration and network issues between the given pod and the given upstream IP. + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-ip 10.4.6.160 + + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ listener for upstream "backend" found + ✓ route for upstream "backend" found + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ``` + +The following example troubleshoots the Consul service mesh configuration and network issues between the given pod and the given upstream. + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-envoy-id db + + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ! no listener for upstream "db" found + ! no route for upstream "backend" found + ! no cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "db" found + ! no healthy endpoints for cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "db" found + ``` + ### `uninstall` The `uninstall` command removes Consul from Kubernetes. diff --git a/website/data/commands-nav-data.json b/website/data/commands-nav-data.json index 62ed01a40048..aac55d7952bb 100644 --- a/website/data/commands-nav-data.json +++ b/website/data/commands-nav-data.json @@ -528,6 +528,23 @@ } ] }, + { + "title": "troubleshoot", + "routes": [ + { + "title": "Overview", + "path": "troubleshoot" + }, + { + "title": "upstreams", + "path": "troubleshoot/upstreams" + }, + { + "title": "proxy", + "path": "troubleshoot/proxy" + } + ] + }, { "title": "validate", "path": "validate" From 1241135e68f7c86b43c5ec79c6044551055268a4 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 14:34:04 -0500 Subject: [PATCH 017/421] Backport of [API Gateway] Validate listener name is not empty into release/1.15.x (#16341) * backport of commit cabcb052fd735c51a63be9e534e7c8bc3dea111d * backport of commit 89baddca259d38dd37e1e9aa3e58170d5067da66 --------- Co-authored-by: Andrew Stucki --- agent/structs/config_entry_gateways.go | 11 ++++-- agent/structs/config_entry_gateways_test.go | 35 +++++++++++++++++++ api/config_entry_gateways.go | 5 ++- .../case-api-gateway-tcp-conflicted/setup.sh | 1 + 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index c757dce984f2..885a301fc467 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -2,6 +2,7 @@ package structs import ( "fmt" + "regexp" "sort" "strings" @@ -778,9 +779,14 @@ func (e *APIGatewayConfigEntry) Validate() error { return e.validateListeners() } +var listenerNameRegex = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`) + func (e *APIGatewayConfigEntry) validateListenerNames() error { listeners := make(map[string]struct{}) for _, listener := range e.Listeners { + if len(listener.Name) < 1 || !listenerNameRegex.MatchString(listener.Name) { + return fmt.Errorf("listener name %q is invalid, must be at least 1 character and contain only letters, numbers, or dashes", listener.Name) + } if _, found := listeners[listener.Name]; found { return fmt.Errorf("found multiple listeners with the name %q", listener.Name) } @@ -861,9 +867,8 @@ const ( // APIGatewayListener represents an individual listener for an APIGateway type APIGatewayListener struct { - // Name is the optional name of the listener in a given gateway. This is - // optional but must be unique within a gateway; therefore, if a gateway - // has more than a single listener, all but one must specify a Name. + // Name is the name of the listener in a given gateway. This must be + // unique within a gateway. Name string // Hostname is the host name that a listener should be bound to. If // unspecified, the listener accepts requests for all hostnames. diff --git a/agent/structs/config_entry_gateways_test.go b/agent/structs/config_entry_gateways_test.go index dd1f624ecad5..ca68ea4f40a5 100644 --- a/agent/structs/config_entry_gateways_test.go +++ b/agent/structs/config_entry_gateways_test.go @@ -1143,12 +1143,40 @@ func TestAPIGateway_Listeners(t *testing.T) { }, validateErr: "multiple listeners with the name", }, + "empty listener name": { + entry: &APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: "api-gw-one", + Listeners: []APIGatewayListener{ + { + Port: 80, + Protocol: "tcp", + }, + }, + }, + validateErr: "listener name \"\" is invalid, must be at least 1 character and contain only letters, numbers, or dashes", + }, + "invalid listener name": { + entry: &APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: "api-gw-one", + Listeners: []APIGatewayListener{ + { + Port: 80, + Protocol: "tcp", + Name: "/", + }, + }, + }, + validateErr: "listener name \"/\" is invalid, must be at least 1 character and contain only letters, numbers, or dashes", + }, "merged listener protocol conflict": { entry: &APIGatewayConfigEntry{ Kind: "api-gateway", Name: "api-gw-two", Listeners: []APIGatewayListener{ { + Name: "listener-one", Port: 80, Protocol: ListenerProtocolHTTP, }, @@ -1167,6 +1195,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-three", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "host.one", }, @@ -1185,6 +1214,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-four", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "host.one", Protocol: APIGatewayListenerProtocol("UDP"), @@ -1199,6 +1229,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-five", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "host.one", Protocol: APIGatewayListenerProtocol("tcp"), @@ -1213,6 +1244,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-six", Listeners: []APIGatewayListener{ { + Name: "listener", Port: -1, Protocol: APIGatewayListenerProtocol("tcp"), }, @@ -1226,6 +1258,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-seven", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "*.*.host.one", Protocol: APIGatewayListenerProtocol("http"), @@ -1240,6 +1273,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-eight", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "host.one", Protocol: APIGatewayListenerProtocol("http"), @@ -1259,6 +1293,7 @@ func TestAPIGateway_Listeners(t *testing.T) { Name: "api-gw-nine", Listeners: []APIGatewayListener{ { + Name: "listener", Port: 80, Hostname: "host.one", Protocol: APIGatewayListenerProtocol("http"), diff --git a/api/config_entry_gateways.go b/api/config_entry_gateways.go index 209c7ae0df98..05e43832c1fd 100644 --- a/api/config_entry_gateways.go +++ b/api/config_entry_gateways.go @@ -268,9 +268,8 @@ func (g *APIGatewayConfigEntry) GetModifyIndex() uint64 { return g.ModifyInd // APIGatewayListener represents an individual listener for an APIGateway type APIGatewayListener struct { - // Name is the optional name of the listener in a given gateway. This is - // optional, however, it must be unique. Therefore, if a gateway has more - // than a single listener, all but one must specify a Name. + // Name is the name of the listener in a given gateway. This must be + // unique within a gateway. Name string // Hostname is the host name that a listener should be bound to, if // unspecified, the listener accepts requests for all hostnames. diff --git a/test/integration/connect/envoy/case-api-gateway-tcp-conflicted/setup.sh b/test/integration/connect/envoy/case-api-gateway-tcp-conflicted/setup.sh index 2394ab23621e..3a45aab7d0ca 100644 --- a/test/integration/connect/envoy/case-api-gateway-tcp-conflicted/setup.sh +++ b/test/integration/connect/envoy/case-api-gateway-tcp-conflicted/setup.sh @@ -9,6 +9,7 @@ listeners = [ { port = 9999 protocol = "tcp" + name = "listener" } ] ' From de2bcabdb81a13220578dc5e43b52492e4520b31 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 15:23:59 -0500 Subject: [PATCH 018/421] Backport of Docs/reformat service splitters conf entry into release/1.15.x (#16283) * backport of commit 99012ae501addac6ac4ec051bfd24bfd5df3db56 * backport of commit 0ffebd8f89cca65dafe24abd27b3af781733880b * backport of commit 352d079872eee969b168d1cdc9718ffe65307362 * backport of commit 6b402e9c444797139cce15e3f9a8206dd21c1f9e * backport of commit 9da4a1fe0e2d015d4e84b70d8746f2c03d11abcf * backport of commit fabc679309d00a4a16f997138a8908fb3f111a81 * backport of commit b57075ec112fb0841066ef451645ff443ac93608 * backport of commit d3cc6ee9e975824e6f061e5949742536c8250945 * Docs/reformat service splitters conf entry (#16264) * for tab testing * updates * Update * adding sandbox to test conf ref types * testing tweaks to the conf ref template * reintroduce tabbed specification * applied feedback from MKO session * applied feedback on format from luke and jared * Apply suggestions from code review Co-authored-by: Dan Upton * fixed some minor HCL formatting in complete conf * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * fixed bad link * resolving conflicts --------- Co-authored-by: boruszak Co-authored-by: Dan Upton Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> resolved cherry-pick conflicts * trying to get rid of conflicts --------- Co-authored-by: boruszak Co-authored-by: trujillo-adam Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --- .../config-entries/service-splitter.mdx | 835 +++++++++++++----- 1 file changed, 630 insertions(+), 205 deletions(-) diff --git a/website/content/docs/connect/config-entries/service-splitter.mdx b/website/content/docs/connect/config-entries/service-splitter.mdx index 4386fba281bc..34ea9597e218 100644 --- a/website/content/docs/connect/config-entries/service-splitter.mdx +++ b/website/content/docs/connect/config-entries/service-splitter.mdx @@ -1,54 +1,575 @@ ---- +--- layout: docs -page_title: Service Splitter - Configuration Entry Reference -description: >- - The service splitter configuration entry kind defines how to divide service mesh traffic between service instances. Use the reference guide to learn about `""service-splitter""` config entry parameters and how it can be used for traffic management behaviors like canary rollouts, blue green deployment, and load balancing across environments. +page_title: Service Splitter Configuration Entry Reference +description: >- + Service splitter configuration entries are L7 traffic management tools for redirecting requests for a service to + multiple instances. Learn how to write `service-splitter` config entries in HCL or YAML with a specification + reference, configuration model, a complete example, and example code by use case. --- -# Service Splitter Configuration Entry +# Service Splitter Configuration Reference + +This reference page describes the structure and contents of service splitter configuration entries. Configure and apply service splitters to redirect a percentage of incoming traffic requests for a service to one or more specific service instances. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in a service splitter configuration entry. Click on a property name to view additional details, including default values. + + + + + +- [`Kind`](#kind): string | required +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string +- [`Partition`](#partition): string +- [`Meta`](#meta): map +- [`Splits`](#splits): map | required + - [`Weight`](#splits-weight): number | required + - [`Service`](#splits-service): string | required + - [`ServiceSubset`](#splits-servicesubset): string + - [`Namespace`](#splits-namespace): string + - [`Partition`](#splits-partition): string + - [`RequestHeaders`](#splits-requestheaders): map + - [`Add`](#splits-requestheaders): map + - [`Set`](#splits-requestheaders): map + - [`Remove`](#splits-requestheaders): map + - [`ResponseHeaders`](#splits-responseheaders): map + - [`Add`](#splits-responseheaders): map + - [`Set`](#splits-responseheaders): map + - [`Remove`](#splits-responseheaders): map + + + + + +- [`apiVersion`](#apiversion): string | required +- [`kind`](#kind): string | required +- [`metadata`](#metadata): object | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | optional +- [`spec`](#spec): object | required + - [`splits`](#spec-splits): list | required + - [`weight`](#spec-splits-weight): float32 | required + - [`service`](#spec-splits-service): string | required + - [`serviceSubset`](#spec-splits-servicesubset): string + - [`namespace`](#spec-splits-namespace): string + - [`partition`](#spec-splits-partition): string + - [`requestHeaders`](#spec-splits-requestheaders): HTTPHeaderModifiers + - [`add`](#spec-splits-requestheaders): map + - [`set`](#spec-splits-requestheaders): map + - [`remove`](#spec-splits-requestheaders): map + - [`responseHeaders`](#spec-splits-responseheaders): HTTPHeaderModifiers + - [`add`](#spec-splits-responseheaders): map + - [`set`](#spec-splits-responseheaders): map + - [`remove`](#spec-splits-responseheaders): map + + + + +## Complete configuration + +When every field is defined, a service splitter configuration entry has the following form: + + + + + +```hcl +Kind = "service-splitter" ## string | required +Name = "config-entry-name" ## string | required +Namespace = "main" ## string +Partition = "partition" ## string +Meta = { ## map + key = "value" +} +Splits = [ ## list | required + { ## map + Weight = 90 ## number | required + Service = "service" ## string + ServiceSubset = "v1" ## string + Namespace = "target-namespace" ## string + Partition = "target-partition" ## string + RequestHeaders = { ## map + Set = { + "X-Web-Version" : "from-v1" + } + } + ResponseHeaders = { ## map + Set = { + "X-Web-Version" : "to-v1" + } + } + }, + { + Weight = 10 + Service = "service" + ServiceSubset = "v2" + Namespace = "target-namespace" + Partition = "target-partition" + RequestHeaders = { + Set = { + "X-Web-Version" : "from-v2" + } + } + ResponseHeaders = { + Set = { + "X-Web-Version" : "to-v2" + } + } + } +] +``` + + + + + +```json +{ + "Kind" : "service-splitter", ## string | required + "Name" : "config-entry-name", ## string | required + "Namespace" : "main", ## string + "Partition" : "partition", ## string + "Meta" : { ## map + "_key_" : "_value_" + }, + "Splits" : [ ## list | required + { ## map + "Weight" : 90, ## number | required + "Service" : "service", ## string + "ServiceSubset" : "v1", ## string + "Namespace" : "target-namespace", ## string + "Partition" : "target-partition", ## string + "RequestHeaders" : { ## map + "Set" : { + "X-Web-Version": "from-v1" + } + }, + "ResponseHeaders" : { ## map + "Set" : { + "X-Web-Version": "to-v1" + } + } + }, + { + "Weight" : 10, + "Service" : "service", + "ServiceSubset" : "v2", + "Namespace" : "target-namespace", + "Partition" : "target-partition", + "RequestHeaders" : { + "Set" : { + "X-Web-Version": "from-v2" + } + }, + "ResponseHeaders" : { + "Set" : { + "X-Web-Version": "to-v2" + } + } + } + ] +} +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 # string | required +kind: ServiceSplitter # string | required +metadata: # object | required + name: config-entry-name # string | required + namespace: main # string +spec: + splits: # list + - weight: 90 # floating point | required + service: service # string + serviceSubset: v1 # string + namespace: target-namespace # string + partition: target-partition # string + requestHeaders: + set: + x-web-version: from-v1 # string + responseHeaders: + set: + x-web-version: to-v1 # string + - weight: 10 + service: service + serviceSubset: v2 + namespace: target-namespace + partition: target-partition + requestHeaders: + set: + x-web-version: from-v2 + responseHeaders: + set: + x-web-version: to-v2 +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service splitter configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `service-splitter`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: String + + +### `Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to apply the configuration entry. + +#### Values + +- Default: None +- Data type: String + +### `Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to apply the configuration entry. + +#### Values + +- Default: `Default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `Splits` + +Defines how much traffic to send to sets of service instances during a traffic split. + +#### Values + +- Default: None +- This field is required. +- Data type: list of objects that can contain the following fields: + - `Weight`: The sum of weights for a set of service instances must add up to 100. + - `Service`: This field is required. + - `ServiceSubset` + - `Namespace` + - `Partition` + - `RequestHeaders` + - `ResponseHeaders` + +### `Splits[].Weight` + +Specifies the percentage of traffic sent to the set of service instances specified in the [`Service`](#service) field. Each weight must be a floating integer between `0` and `100`. The smallest representable value is `.01`. The sum of weights across all splits must add up to `100`. + +#### Values + +- Default: `null` +- This field is required. +- Data type: Floating number from `.01` to `100`. + +### `Splits[].Service` + +Specifies the name of the service to resolve. + +#### Values + +- Default: Inherits the value of the [`Name`](#name) field. +- Data type: String + +### `Splits[].ServiceSubset` + +Specifies a subset of the service to resolve. A service subset assigns a name to a specific subset of discoverable service instances within a datacenter, such as `version2` or `canary`. All services have an unnamed default subset that returns all healthy instances. + +You can define service subsets in a [service resolver configuration entry](/consul/docs/connect/config-entries/service-resolver), which are referenced by their names throughout the other configuration entries. This field overrides the default subset value in the service resolver configuration entry. + +#### Values + +- Default: If empty, the `split` uses the default subset. +- Data type: String + +### `Splits[].Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to use in the FQDN when resolving the service. + +#### Values + +- Default: Inherits the value of [`Namespace`](#Namespace) from the top-level of the configuration entry. +- Data type: String + +### `Splits[].Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to use in the FQDN when resolving the service. + +#### Values + +- Default: By default, the `service-splitter` uses the [admin partition specified in the top-level configuration entry](#partition). +- Data type: String + +### `Splits[].RequestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `Add`: Map of one or more key-value pairs + - `Set`: Map of one or more key-value pairs + - `Remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + +### `Splits[].ResponseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `Add`: Map of one or more string key-value pairs + - `Set`: Map of one or more string key-value pairs + - `Remove`: Map of one or more string key-value pairs + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + + + + +### `apiVersion` + +Kubernetes-only parameter that specifies the version of the Consul API that the configuration entry maps to Kubernetes configurations. The value must be `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `serviceSplitter`. + +### `metadata.name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Inherits name from the host node +- This field is required. +- Data type: String + + +### `metadata.namespace` + +Specifies the Consul namespace to use for resolving the service. You can map Consul namespaces to Kubernetes Namespaces in different ways. Refer to [Custom Resource Definitions (CRDs) for Consul on Kubernetes](/consul/docs/k8s/crds#consul-enterprise) for additional information. + +#### Values + +- Default: None +- Data type: String + +### `spec` + +Kubernetes-only field that contains all of the configurations for service splitter pods. + +#### Values + +- Default: none +- This field is required. +- Data type: Object containing [`spec.splits`](#spec-splits) configuration + +### `spec.meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `spec.splits` + +Defines how much traffic to send to sets of service instances during a traffic split. --> **v1.8.4+:** On Kubernetes, the `ServiceSplitter` custom resource is supported in Consul versions 1.8.4+.
-**v1.6.0+:** On other platforms, this config entry is supported in Consul versions 1.6.0+. +#### Values -The `service-splitter` config entry kind (`ServiceSplitter` on Kubernetes) controls how to split incoming Connect -requests across different subsets of a single service (like during staged -canary rollouts), or perhaps across different services (like during a v2 -rewrite or other type of codebase migration). +- Default: None +- This field is required. +- Data type: list of objects that can contain the following fields: + - `weight`: The sum of weights for a set of service instances. The total defined value must add up to 100. + - `service`: This field is required. + - `serviceSubset` + - `namespace` + - `partition` + - `requestHeaders` + - `responseHeaders` -If no splitter config is defined for a service it is assumed 100% of traffic -flows to a service with the same name and discovery continues on to the -resolution stage. +### `spec.splits[].weight` -## Interaction with other Config Entries +Specifies the percentage of traffic sent to the set of service instances specified in the [`spec.splits.service`](#spec-splits-service) field. Each weight must be a floating integer between `0` and `100`. The smallest representable value is `.01`. The sum of weights across all splits must add up to `100`. -- Service splitter config entries are a component of [L7 Traffic - Management](/consul/docs/connect/l7-traffic). +#### Values -- Service splitter config entries are restricted to only services that define - their protocol as http-based via a corresponding - [`service-defaults`](/consul/docs/connect/config-entries/service-defaults) config - entry or globally via - [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults) . +- Default: `null` +- This field is required. +- Data type: Floating integer from `.01` to `100` -- Any split destination that specifies a different `Service` field and omits - the `ServiceSubset` field is eligible for further splitting should a splitter - be configured for that other service, otherwise resolution proceeds according - to any configured - [`service-resolver`](/consul/docs/connect/config-entries/service-resolver). +### `spec.splits[].service` -## UI +Specifies the name of the service to resolve. -Once a `service-splitter` is successfully entered, you can view it in the UI. Service routers, service splitters, and service resolvers can all be viewed by clicking on your service then switching to the _routing_ tab. +#### Values -![screenshot of service splitter in the UI](/img/l7-routing/Splitter.png) +- Default: The service matching the configuration entry [`meta.name`](#metadata-name) field. +- Data type: String -## Sample Config Entries +### `spec.splits[].serviceSubset` + +Specifies a subset of the service to resolve. This field overrides the `DefaultSubset`. + +#### Values + +- Default: Inherits the name of the default subset. +- Data type: String + +### `spec.splits[].namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to use when resolving the service. + +#### Values + +- Default: The namespace specified in the top-level configuration entry. +- Data type: String + +### `spec.splits[].partition` + +Specifies which [admin partition](/consul/docs/enterprise/admin-partitions) to use in the FQDN when resolving the service. + +#### Values + +- Default: `default` +- Data type: String + +### `spec.splits[].requestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `add`: Map of one or more key-value pairs + - `set`: Map of one or more key-value pairs + - `remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + +### `spec.splits[].responseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `add`: Map of one or more string key-value pairs + - `set`: Map of one or more string key-value pairs + - `remove`: Map of one or more string key-value pairs + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + + + + +## Examples + +The following examples demonstrate common service splitter configuration patterns for specific use cases. ### Two subsets of same service Split traffic between two subsets of the same service: - + + + ```hcl Kind = "service-splitter" @@ -65,18 +586,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 90 - serviceSubset: v1 - - weight: 10 - serviceSubset: v2 -``` + + + ```json { @@ -95,13 +607,34 @@ spec: } ``` - + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 90 + serviceSubset: v1 + - weight: 10 + serviceSubset: v2 +``` + + + + ### Two different services Split traffic between two services: - + + + ```hcl Kind = "service-splitter" @@ -118,18 +651,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 50 - # will default to service with same name as config entry ("web") - - weight: 50 - service: web-rewrite -``` + + + ```json { @@ -147,14 +671,35 @@ spec: } ``` - + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 50 + # defaults to the service with same name as the configuration entry ("web") + - weight: 50 + service: web-rewrite +``` + + + + ### Set HTTP Headers Split traffic between two subsets with extra headers added so clients can tell which version: - + + + ```hcl Kind = "service-splitter" @@ -181,24 +726,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 90 - serviceSubset: v1 - responseHeaders: - set: - x-web-version: v1 - - weight: 10 - serviceSubset: v2 - responseHeaders: - set: - x-web-version: v2 -``` + + + ```json { @@ -227,136 +757,31 @@ spec: } ``` - + -## Available Fields -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the namespace to which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the admin partition to which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: - 'Specifies arbitrary KV metadata pairs. Added in Consul 1.8.4.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: 'Set to the name of the service being configured.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'Splits', - type: 'array', - description: - 'Defines how much traffic to send to which set of service instances during a traffic split. The sum of weights across all splits must add up to 100.', - children: [ - { - name: 'weight', - type: 'float32: 0', - description: - 'A value between 0 and 100 reflecting what portion of traffic should be directed to this split. The smallest representable weight is 1/10000 or .01%', - }, - { - name: 'Service', - type: 'string: ""', - description: 'The service to resolve instead of the default.', - }, - { - name: 'ServiceSubset', - type: 'string: ""', - description: { - hcl: - "A named subset of the given service to resolve instead of one defined as that service's `DefaultSubset`. If empty the default subset is used.", - yaml: - "A named subset of the given service to resolve instead of one defined as that service's `defaultSubset`. If empty the default subset is used.", - }, - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: - 'The namespace to resolve the service from instead of the current namespace. If empty, the current namespace is used.', - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: - 'The admin partition to resolve the service from instead of the current partition. If empty, the current partition is used.', - }, - { - name: 'RequestHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to requests routed to this split. - This cannot be used with a \`tcp\` listener.`, - }, - { - name: 'ResponseHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to responses from this split. - This cannot be used with a \`tcp\` listener.`, - }, - ], - }, - ]} -/> - -## ACLs -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). + -Reading a `service-splitter` config entry requires `service:read` on the resource. +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 90 + serviceSubset: v1 + responseHeaders: + set: + x-web-version: v1 + - weight: 10 + serviceSubset: v2 + responseHeaders: + set: + x-web-version: v2 +``` -Creating, updating, or deleting a `service-splitter` config entry requires -`service:write` on the resource and `service:read` on any other service referenced by -name in these fields: + -- [`Splits[].Service`](#service) + \ No newline at end of file From 2604a03799669113b9a53b1aabb8ef16e29bf996 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 15:36:42 -0500 Subject: [PATCH 019/421] backport of commit 817d85df1231f33c40bd90e661f68250d018af12 (#16344) Co-authored-by: Derek Menteer --- .changelog/16339.txt | 3 + agent/consul/state/catalog.go | 48 ++++- agent/consul/state/catalog_test.go | 55 +++++- agent/consul/state/peering.go | 185 +++++++++++++----- agent/consul/state/peering_test.go | 102 +++++++++- .../services/peerstream/stream_test.go | 75 ++++--- .../peerstream/subscription_manager.go | 2 +- .../peerstream/subscription_manager_test.go | 29 ++- .../exported_peered_services_test.go | 8 + agent/structs/peering.go | 3 +- agent/structs/structs.go | 6 + 11 files changed, 424 insertions(+), 92 deletions(-) create mode 100644 .changelog/16339.txt diff --git a/.changelog/16339.txt b/.changelog/16339.txt new file mode 100644 index 000000000000..cf44f010aff5 --- /dev/null +++ b/.changelog/16339.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix bug where services were incorrectly imported as connect-enabled. +``` diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 4d645013680a..39a40d9cad36 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -900,12 +900,17 @@ func ensureServiceTxn(tx WriteTxn, idx uint64, node string, preserveIndexes bool return fmt.Errorf("failed updating gateway mapping: %s", err) } + if svc.PeerName == "" && sn.Name != "" { + if err := upsertKindServiceName(tx, idx, structs.ServiceKindConnectEnabled, sn); err != nil { + return fmt.Errorf("failed to persist service name as connect-enabled: %v", err) + } + } + + // Update the virtual IP for the service supported, err := virtualIPsSupported(tx, nil) if err != nil { return err } - - // Update the virtual IP for the service if supported { psn := structs.PeeredServiceName{Peer: svc.PeerName, ServiceName: sn} vip, err := assignServiceVirtualIP(tx, idx, psn) @@ -1964,6 +1969,24 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st return fmt.Errorf("Could not find any service %s: %s", svc.ServiceName, err) } + // Cleanup ConnectEnabled for this service if none exist. + if svc.PeerName == "" && (svc.ServiceKind == structs.ServiceKindConnectProxy || svc.ServiceConnect.Native) { + service := svc.ServiceName + if svc.ServiceKind == structs.ServiceKindConnectProxy { + service = svc.ServiceProxy.DestinationServiceName + } + sn := structs.ServiceName{Name: service, EnterpriseMeta: svc.EnterpriseMeta} + connectEnabled, err := serviceHasConnectEnabledInstances(tx, sn.Name, &sn.EnterpriseMeta) + if err != nil { + return fmt.Errorf("failed to search for connect instances for service %q: %w", sn.Name, err) + } + if !connectEnabled { + if err := cleanupKindServiceName(tx, idx, sn, structs.ServiceKindConnectEnabled); err != nil { + return fmt.Errorf("failed to cleanup connect-enabled service name: %v", err) + } + } + } + if svc.PeerName == "" { sn := structs.ServiceName{Name: svc.ServiceName, EnterpriseMeta: svc.EnterpriseMeta} if err := cleanupGatewayWildcards(tx, idx, sn, false); err != nil { @@ -3731,6 +3754,27 @@ func serviceHasConnectInstances(tx WriteTxn, serviceName string, entMeta *acl.En return hasConnectInstance, hasNonConnectInstance, nil } +// serviceHasConnectEnabledInstances returns whether the given service name +// has a corresponding connect-proxy or connect-native instance. +// This function is mostly a clone of `serviceHasConnectInstances`, but it has +// an early return to improve performance and returns true if at least one +// connect-native instance exists. +func serviceHasConnectEnabledInstances(tx WriteTxn, serviceName string, entMeta *acl.EnterpriseMeta) (bool, error) { + query := Query{ + Value: serviceName, + EnterpriseMeta: *entMeta, + } + + svc, err := tx.First(tableServices, indexConnect, query) + if err != nil { + return false, fmt.Errorf("failed service lookup: %w", err) + } + if svc != nil { + return true, nil + } + return false, nil +} + // updateGatewayService associates services with gateways after an eligible event // ie. Registering a service in a namespace targeted by a gateway func updateGatewayService(tx WriteTxn, idx uint64, mapping *structs.GatewayService) error { diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index d354b9b094e3..cef5ba0a0a07 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -8664,7 +8664,7 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { }, } - var idx uint64 + var idx, connectEnabledIdx uint64 testRegisterNode(t, s, idx, "node1") for _, svc := range services { @@ -8678,7 +8678,28 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { require.Len(t, gotNames, 1) require.Equal(t, svc.CompoundServiceName(), gotNames[0].Service) require.Equal(t, svc.Kind, gotNames[0].Kind) + if svc.Kind == structs.ServiceKindConnectProxy { + connectEnabledIdx = idx + } + } + + // A ConnectEnabled service should exist if a corresponding ConnectProxy or ConnectNative service exists. + verifyConnectEnabled := func(expectIdx uint64) { + gotIdx, gotNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindConnectEnabled) + require.NoError(t, err) + require.Equal(t, expectIdx, gotIdx) + require.Equal(t, []*KindServiceName{ + { + Kind: structs.ServiceKindConnectEnabled, + Service: structs.NewServiceName("foo", entMeta), + RaftIndex: structs.RaftIndex{ + CreateIndex: connectEnabledIdx, + ModifyIndex: connectEnabledIdx, + }, + }, + }, gotNames) } + verifyConnectEnabled(connectEnabledIdx) // Register another ingress gateway and there should be two names under the kind index newIngress := structs.NodeService{ @@ -8749,6 +8770,38 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { require.NoError(t, err) require.Equal(t, idx, gotIdx) require.Empty(t, got) + + // A ConnectEnabled entry should not be removed until all corresponding services are removed. + { + verifyConnectEnabled(connectEnabledIdx) + // Add a connect-native service. + idx++ + require.NoError(t, s.EnsureService(idx, "node1", &structs.NodeService{ + Kind: structs.ServiceKindTypical, + ID: "foo", + Service: "foo", + Address: "5.5.5.5", + Port: 5555, + EnterpriseMeta: *entMeta, + Connect: structs.ServiceConnect{ + Native: true, + }, + })) + verifyConnectEnabled(connectEnabledIdx) + + // Delete the proxy. This should not clean up the entry, because we still have a + // connect-native service registered. + idx++ + require.NoError(t, s.DeleteService(idx, "node1", "connect-proxy", entMeta, "")) + verifyConnectEnabled(connectEnabledIdx) + + // Remove the connect-native service to clear out the connect-enabled entry. + require.NoError(t, s.DeleteService(idx, "node1", "foo", entMeta, "")) + gotIdx, gotNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindConnectEnabled) + require.NoError(t, err) + require.Equal(t, idx, gotIdx) + require.Empty(t, gotNames) + } } func assertMaxIndexes(t *testing.T, tx ReadTxn, expect map[string]uint64, skip ...string) { diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index 32e604500449..3fafb98cba7a 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -770,88 +770,181 @@ func exportedServicesForPeerTxn( maxIdx := peering.ModifyIndex entMeta := structs.NodeEnterpriseMetaInPartition(peering.Partition) - idx, conf, err := getExportedServicesConfigEntryTxn(tx, ws, nil, entMeta) + idx, exportConf, err := getExportedServicesConfigEntryTxn(tx, ws, nil, entMeta) if err != nil { return 0, nil, fmt.Errorf("failed to fetch exported-services config entry: %w", err) } if idx > maxIdx { maxIdx = idx } - if conf == nil { + if exportConf == nil { return maxIdx, &structs.ExportedServiceList{}, nil } var ( - normalSet = make(map[structs.ServiceName]struct{}) - discoSet = make(map[structs.ServiceName]struct{}) + // exportedServices will contain the listing of all service names that are being exported + // and will need to be queried for connect / discovery chain information. + exportedServices = make(map[structs.ServiceName]struct{}) + + // exportedConnectServices will contain the listing of all connect service names that are being exported. + exportedConnectServices = make(map[structs.ServiceName]struct{}) + + // namespaceConnectServices provides a listing of all connect service names for a particular partition+namespace pair. + namespaceConnectServices = make(map[acl.EnterpriseMeta]map[string]struct{}) + + // namespaceDiscoChains provides a listing of all disco chain names for a particular partition+namespace pair. + namespaceDiscoChains = make(map[acl.EnterpriseMeta]map[string]struct{}) ) - // At least one of the following should be true for a name for it to - // replicate: - // - // - are a discovery chain by definition (service-router, service-splitter, service-resolver) - // - have an explicit sidecar kind=connect-proxy - // - use connect native mode + // Helper function for inserting data and auto-creating maps. + insertEntry := func(m map[acl.EnterpriseMeta]map[string]struct{}, entMeta acl.EnterpriseMeta, name string) { + names, ok := m[entMeta] + if !ok { + names = make(map[string]struct{}) + m[entMeta] = names + } + names[name] = struct{}{} + } - for _, svc := range conf.Services { + // Build the set of all services that will be exported. + // Any possible namespace wildcards or "consul" services should be removed by this step. + for _, svc := range exportConf.Services { // Prevent exporting the "consul" service. if svc.Name == structs.ConsulServiceName { continue } - svcMeta := acl.NewEnterpriseMetaWithPartition(entMeta.PartitionOrDefault(), svc.Namespace) + svcEntMeta := acl.NewEnterpriseMetaWithPartition(entMeta.PartitionOrDefault(), svc.Namespace) + svcName := structs.NewServiceName(svc.Name, &svcEntMeta) - sawPeer := false + peerFound := false for _, consumer := range svc.Consumers { - name := structs.NewServiceName(svc.Name, &svcMeta) - - if _, ok := normalSet[name]; ok { - // Service was covered by a wildcard that was already accounted for - continue + if consumer.Peer == peering.Name { + peerFound = true + break } - if consumer.Peer != peering.Name { - continue + } + // Only look for more information if the matching peer was found. + if !peerFound { + continue + } + + // If this isn't a wildcard, we can simply add it to the list of services to watch and move to the next entry. + if svc.Name != structs.WildcardSpecifier { + exportedServices[svcName] = struct{}{} + continue + } + + // If all services in the namespace are exported by the wildcard, query those service names. + idx, typicalServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindTypical, svcEntMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get typical service names: %w", err) + } + if idx > maxIdx { + maxIdx = idx + } + for _, sn := range typicalServices { + // Prevent exporting the "consul" service. + if sn.Service.Name != structs.ConsulServiceName { + exportedServices[sn.Service] = struct{}{} } - sawPeer = true + } - if svc.Name != structs.WildcardSpecifier { - normalSet[name] = struct{}{} + // List all config entries of kind service-resolver, service-router, service-splitter, because they + // will be exported as connect services. + idx, discoChains, err := listDiscoveryChainNamesTxn(tx, ws, nil, svcEntMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get discovery chain names: %w", err) + } + if idx > maxIdx { + maxIdx = idx + } + for _, sn := range discoChains { + // Prevent exporting the "consul" service. + if sn.Name != structs.ConsulServiceName { + exportedConnectServices[sn] = struct{}{} + insertEntry(namespaceDiscoChains, svcEntMeta, sn.Name) } } + } - // If the target peer is a consumer, and all services in the namespace are exported, query those service names. - if sawPeer && svc.Name == structs.WildcardSpecifier { - idx, typicalServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindTypical, svcMeta) + // At least one of the following should be true for a name to replicate it as a *connect* service: + // - are a discovery chain by definition (service-router, service-splitter, service-resolver) + // - have an explicit sidecar kind=connect-proxy + // - use connect native mode + // - are registered with a terminating gateway + populateConnectService := func(sn structs.ServiceName) error { + // Load all disco-chains in this namespace if we haven't already. + if _, ok := namespaceDiscoChains[sn.EnterpriseMeta]; !ok { + // Check to see if we have a discovery chain with the same name. + idx, chains, err := listDiscoveryChainNamesTxn(tx, ws, nil, sn.EnterpriseMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to get typical service names: %w", err) + return fmt.Errorf("failed to get connect services: %w", err) } if idx > maxIdx { maxIdx = idx } - for _, s := range typicalServices { - // Prevent exporting the "consul" service. - if s.Service.Name == structs.ConsulServiceName { - continue - } - normalSet[s.Service] = struct{}{} + for _, sn := range chains { + insertEntry(namespaceDiscoChains, sn.EnterpriseMeta, sn.Name) } + } + // Check to see if we have the connect service. + if _, ok := namespaceDiscoChains[sn.EnterpriseMeta][sn.Name]; ok { + exportedConnectServices[sn] = struct{}{} + // Do not early return because we have multiple watches that should be established. + } - // list all config entries of kind service-resolver, service-router, service-splitter? - idx, discoChains, err := listDiscoveryChainNamesTxn(tx, ws, nil, svcMeta) + // Load all services in this namespace if we haven't already. + if _, ok := namespaceConnectServices[sn.EnterpriseMeta]; !ok { + // This is more efficient than querying the service instance table. + idx, connectServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindConnectEnabled, sn.EnterpriseMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to get discovery chain names: %w", err) + return fmt.Errorf("failed to get connect services: %w", err) } if idx > maxIdx { maxIdx = idx } - for _, sn := range discoChains { - discoSet[sn] = struct{}{} + for _, ksn := range connectServices { + insertEntry(namespaceConnectServices, sn.EnterpriseMeta, ksn.Service.Name) } } + // Check to see if we have the connect service. + if _, ok := namespaceConnectServices[sn.EnterpriseMeta][sn.Name]; ok { + exportedConnectServices[sn] = struct{}{} + // Do not early return because we have multiple watches that should be established. + } + + // Check if the service is exposed via terminating gateways. + svcGateways, err := tx.Get(tableGatewayServices, indexService, sn) + if err != nil { + return fmt.Errorf("failed gateway lookup for %q: %w", sn.Name, err) + } + ws.Add(svcGateways.WatchCh()) + for svc := svcGateways.Next(); svc != nil; svc = svcGateways.Next() { + gs, ok := svc.(*structs.GatewayService) + if !ok { + return fmt.Errorf("failed converting to GatewayService for %q", sn.Name) + } + if gs.GatewayKind == structs.ServiceKindTerminatingGateway { + exportedConnectServices[sn] = struct{}{} + break + } + } + + return nil } - normal := maps.SliceOfKeys(normalSet) - disco := maps.SliceOfKeys(discoSet) + // Perform queries and check if each service is connect-enabled. + for sn := range exportedServices { + // Do not query for data if we already know it's a connect service. + if _, ok := exportedConnectServices[sn]; ok { + continue + } + if err := populateConnectService(sn); err != nil { + return 0, nil, err + } + } + // Fetch the protocol / targets for connect services. chainInfo := make(map[structs.ServiceName]structs.ExportedDiscoveryChainInfo) populateChainInfo := func(svc structs.ServiceName) error { if _, ok := chainInfo[svc]; ok { @@ -899,21 +992,17 @@ func exportedServicesForPeerTxn( return nil } - for _, svc := range normal { - if err := populateChainInfo(svc); err != nil { - return 0, nil, err - } - } - for _, svc := range disco { + for svc := range exportedConnectServices { if err := populateChainInfo(svc); err != nil { return 0, nil, err } } - structs.ServiceList(normal).Sort() + sortedServices := maps.SliceOfKeys(exportedServices) + structs.ServiceList(sortedServices).Sort() list := &structs.ExportedServiceList{ - Services: normal, + Services: sortedServices, DiscoChains: chainInfo, } diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 2c2caaab9a97..81933500d2f2 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -1908,18 +1908,28 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { }, }, { + // Should be exported as both a normal and disco chain (resolver). Name: "mysql", Consumers: []structs.ServiceConsumer{ {Peer: "my-peering"}, }, }, { + // Should be exported as both a normal and disco chain (connect-proxy). Name: "redis", Consumers: []structs.ServiceConsumer{ {Peer: "my-peering"}, }, }, { + // Should only be exported as a normal service. + Name: "prometheus", + Consumers: []structs.ServiceConsumer{ + {Peer: "my-peering"}, + }, + }, + { + // Should not be exported (different peer consumer) Name: "mongo", Consumers: []structs.ServiceConsumer{ {Peer: "my-other-peering"}, @@ -1932,12 +1942,37 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { require.True(t, watchFired(ws)) ws = memdb.NewWatchSet() + // Register extra things so that disco chain entries appear. + lastIdx++ + require.NoError(t, s.EnsureNode(lastIdx, &structs.Node{ + Node: "node1", Address: "10.0.0.1", + })) + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "node1", &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "redis-sidecar-proxy", + Service: "redis-sidecar-proxy", + Port: 5005, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "redis", + }, + })) + ensureConfigEntry(t, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "mysql", + EnterpriseMeta: *defaultEntMeta, + }) + expect := &structs.ExportedServiceList{ Services: []structs.ServiceName{ { Name: "mysql", EnterpriseMeta: *defaultEntMeta, }, + { + Name: "prometheus", + EnterpriseMeta: *defaultEntMeta, + }, { Name: "redis", EnterpriseMeta: *defaultEntMeta, @@ -1998,17 +2033,21 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { ws = memdb.NewWatchSet() expect := &structs.ExportedServiceList{ + // Only "billing" shows up, because there are no other service instances running, + // and "consul" is never exported. Services: []structs.ServiceName{ { Name: "billing", EnterpriseMeta: *defaultEntMeta, }, }, + // Only "mysql" appears because there it has a service resolver. + // "redis" does not appear, because it's a sidecar proxy without a corresponding service, so the wildcard doesn't find it. DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ - newSN("billing"): { + newSN("mysql"): { Protocol: "tcp", TCPTargets: []*structs.DiscoveryTarget{ - newTarget("billing", "", "dc1"), + newTarget("mysql", "", "dc1"), }, }, }, @@ -2025,13 +2064,17 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { ID: "payments", Service: "payments", Port: 5000, })) - // The proxy will be ignored. + // The proxy will cause "payments" to be output in the disco chains. It will NOT be output + // in the normal services list. lastIdx++ require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ Kind: structs.ServiceKindConnectProxy, ID: "payments-proxy", Service: "payments-proxy", Port: 5000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "payments", + }, })) lastIdx++ // The consul service should never be exported. @@ -2099,10 +2142,11 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { }, DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ // NOTE: no consul-redirect here - newSN("billing"): { + // NOTE: no billing here, because it does not have a proxy. + newSN("payments"): { Protocol: "http", }, - newSN("payments"): { + newSN("mysql"): { Protocol: "http", }, newSN("resolver"): { @@ -2129,6 +2173,9 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { lastIdx++ require.NoError(t, s.DeleteConfigEntry(lastIdx, structs.ServiceSplitter, "splitter", nil)) + lastIdx++ + require.NoError(t, s.DeleteConfigEntry(lastIdx, structs.ServiceResolver, "mysql", nil)) + require.True(t, watchFired(ws)) ws = memdb.NewWatchSet() @@ -2160,6 +2207,51 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { require.Equal(t, expect, got) }) + testutil.RunStep(t, "terminating gateway services are exported", func(t *testing.T) { + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ + ID: "term-svc", Service: "term-svc", Port: 6000, + })) + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + Service: "some-terminating-gateway", + ID: "some-terminating-gateway", + Port: 9000, + })) + lastIdx++ + require.NoError(t, s.EnsureConfigEntry(lastIdx, &structs.TerminatingGatewayConfigEntry{ + Kind: structs.TerminatingGateway, + Name: "some-terminating-gateway", + Services: []structs.LinkedService{{Name: "term-svc"}}, + })) + + expect := &structs.ExportedServiceList{ + Services: []structs.ServiceName{ + newSN("payments"), + newSN("term-svc"), + }, + DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ + newSN("payments"): { + Protocol: "http", + }, + newSN("resolver"): { + Protocol: "http", + }, + newSN("router"): { + Protocol: "http", + }, + newSN("term-svc"): { + Protocol: "http", + }, + }, + } + idx, got, err := s.ExportedServicesForPeer(ws, id, "dc1") + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Equal(t, expect, got) + }) + testutil.RunStep(t, "deleting the config entry clears exported services", func(t *testing.T) { expect := &structs.ExportedServiceList{} diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index b21f35bccef1..a0a4e9292ed7 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -844,6 +844,13 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { Node: &structs.Node{Node: "foo", Address: "10.0.0.1"}, Service: &structs.NodeService{ID: "mysql-1", Service: "mysql", Port: 5000}, } + mysqlSidecar := &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mysql-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mysql", + }, + } lastIdx++ require.NoError(t, store.EnsureNode(lastIdx, mysql.Node)) @@ -851,6 +858,9 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { lastIdx++ require.NoError(t, store.EnsureService(lastIdx, "foo", mysql.Service)) + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "foo", mysqlSidecar)) + mongoSvcDefaults := &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, Name: "mongo", @@ -870,6 +880,24 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { mysqlProxySN = structs.NewServiceName("mysql-sidecar-proxy", nil).String() ) + testutil.RunStep(t, "initial stream data is received", func(t *testing.T) { + expectReplEvents(t, client, + func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { + require.Equal(t, pbpeerstream.TypeURLPeeringTrustBundle, msg.GetResponse().ResourceURL) + // Roots tested in TestStreamResources_Server_CARootUpdates + }, + func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { + require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) + require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) + require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) + + var exportedServices pbpeerstream.ExportedServiceList + require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) + require.ElementsMatch(t, []string{}, exportedServices.Services) + }, + ) + }) + testutil.RunStep(t, "exporting mysql leads to an UPSERT event", func(t *testing.T) { entry := &structs.ExportedServicesConfigEntry{ Name: "default", @@ -895,10 +923,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, store.EnsureConfigEntry(lastIdx, entry)) expectReplEvents(t, client, - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - require.Equal(t, pbpeerstream.TypeURLPeeringTrustBundle, msg.GetResponse().ResourceURL) - // Roots tested in TestStreamResources_Server_CARootUpdates - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { // no mongo instances exist require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) @@ -909,16 +933,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) require.Len(t, nodes.Nodes, 0) }, - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - // proxies can't export because no mesh gateway exists yet - require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) - - var nodes pbpeerstream.ExportedService - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) - require.Len(t, nodes.Nodes, 0) - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) require.Equal(t, mysqlSN, msg.GetResponse().ResourceID) @@ -938,17 +952,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) require.Len(t, nodes.Nodes, 0) }, - // This event happens because this is the first test case and there are - // no exported services when replication is initially set up. - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) - require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) - - var exportedServices pbpeerstream.ExportedServiceList - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) - require.ElementsMatch(t, []string{}, exportedServices.Services) - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) @@ -978,7 +981,7 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { expectReplEvents(t, client, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) + require.Equal(t, mysqlProxySN, msg.GetResponse().ResourceID) require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var nodes pbpeerstream.ExportedService @@ -986,16 +989,26 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.Len(t, nodes.Nodes, 1) pm := nodes.Nodes[0].Service.Connect.PeerMeta - require.Equal(t, "grpc", pm.Protocol) + require.Equal(t, "tcp", pm.Protocol) spiffeIDs := []string{ - "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mongo", + "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mysql", "spiffe://11111111-2222-3333-4444-555555555555.consul/gateway/mesh/dc/dc1", } require.Equal(t, spiffeIDs, pm.SpiffeID) }, + ) + }) + + testutil.RunStep(t, "register service resolver to send proxy updates", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "mongo", + })) + expectReplEvents(t, client, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mysqlProxySN, msg.GetResponse().ResourceID) + require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var nodes pbpeerstream.ExportedService @@ -1003,9 +1016,9 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.Len(t, nodes.Nodes, 1) pm := nodes.Nodes[0].Service.Connect.PeerMeta - require.Equal(t, "tcp", pm.Protocol) + require.Equal(t, "grpc", pm.Protocol) spiffeIDs := []string{ - "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mysql", + "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mongo", "spiffe://11111111-2222-3333-4444-555555555555.consul/gateway/mesh/dc/dc1", } require.Equal(t, spiffeIDs, pm.SpiffeID) diff --git a/agent/grpc-external/services/peerstream/subscription_manager.go b/agent/grpc-external/services/peerstream/subscription_manager.go index 0f9174dd8599..dea33a551fcc 100644 --- a/agent/grpc-external/services/peerstream/subscription_manager.go +++ b/agent/grpc-external/services/peerstream/subscription_manager.go @@ -143,7 +143,7 @@ func (m *subscriptionManager) handleEvent(ctx context.Context, state *subscripti pending := &pendingPayload{} m.syncNormalServices(ctx, state, evt.Services) if m.config.ConnectEnabled { - m.syncDiscoveryChains(state, pending, evt.ListAllDiscoveryChains()) + m.syncDiscoveryChains(state, pending, evt.DiscoChains) } err := pending.Add( diff --git a/agent/grpc-external/services/peerstream/subscription_manager_test.go b/agent/grpc-external/services/peerstream/subscription_manager_test.go index 6d7a41979ce3..c7b77edec961 100644 --- a/agent/grpc-external/services/peerstream/subscription_manager_test.go +++ b/agent/grpc-external/services/peerstream/subscription_manager_test.go @@ -472,15 +472,40 @@ func TestSubscriptionManager_InitialSnapshot(t *testing.T) { Node: &structs.Node{Node: "foo", Address: "10.0.0.1"}, Service: &structs.NodeService{ID: "mysql-1", Service: "mysql", Port: 5000}, } + mysqlSidecar := structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mysql-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mysql", + }, + } backend.ensureNode(t, mysql.Node) backend.ensureService(t, "foo", mysql.Service) + backend.ensureService(t, "foo", &mysqlSidecar) mongo := &structs.CheckServiceNode{ - Node: &structs.Node{Node: "zip", Address: "10.0.0.3"}, - Service: &structs.NodeService{ID: "mongo-1", Service: "mongo", Port: 5000}, + Node: &structs.Node{Node: "zip", Address: "10.0.0.3"}, + Service: &structs.NodeService{ + ID: "mongo-1", + Service: "mongo", + Port: 5000, + }, + } + mongoSidecar := structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mongo-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mongo", + }, } backend.ensureNode(t, mongo.Node) backend.ensureService(t, "zip", mongo.Service) + backend.ensureService(t, "zip", &mongoSidecar) + + backend.ensureConfigEntry(t, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "chain", + }) var ( mysqlCorrID = subExportedService + structs.NewServiceName("mysql", nil).String() diff --git a/agent/proxycfg-glue/exported_peered_services_test.go b/agent/proxycfg-glue/exported_peered_services_test.go index 8ba7390fbf5c..77341e515776 100644 --- a/agent/proxycfg-glue/exported_peered_services_test.go +++ b/agent/proxycfg-glue/exported_peered_services_test.go @@ -49,6 +49,14 @@ func TestServerExportedPeeredServices(t *testing.T) { }, })) + // Create resolvers for each of the services so that they are guaranteed to be replicated by the peer stream. + for _, s := range []string{"web", "api", "db"} { + require.NoError(t, store.EnsureConfigEntry(0, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: s, + })) + } + authz := policyAuthorizer(t, ` service "web" { policy = "read" } service "api" { policy = "read" } diff --git a/agent/structs/peering.go b/agent/structs/peering.go index 52d2f32a3786..714a442e8f77 100644 --- a/agent/structs/peering.go +++ b/agent/structs/peering.go @@ -62,8 +62,7 @@ func (i ExportedDiscoveryChainInfo) Equal(o ExportedDiscoveryChainInfo) bool { return true } -// ListAllDiscoveryChains returns all discovery chains (union of Services and -// DiscoChains). +// ListAllDiscoveryChains returns all discovery chains (union of Services and DiscoChains). func (list *ExportedServiceList) ListAllDiscoveryChains() map[ServiceName]ExportedDiscoveryChainInfo { chainsByName := make(map[ServiceName]ExportedDiscoveryChainInfo) if list == nil { diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 0a95da695ae4..5191272a645f 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -1235,6 +1235,12 @@ const ( // This service allows external traffic to exit the mesh through a terminating gateway // based on centralized configuration. ServiceKindDestination ServiceKind = "destination" + + // ServiceKindConnectEnabled is used to indicate whether a service is either + // connect-native or if the service has a corresponding sidecar. It is used for + // internal query purposes and should not be exposed to users as a valid Kind + // option. + ServiceKindConnectEnabled ServiceKind = "connect-enabled" ) // Type to hold a address and port of a service From 9f29e0c4fc5d17c70da488d77eb99f498e201338 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 22:33:18 -0500 Subject: [PATCH 020/421] backport of commit 326d5c02c608e5658c7ee1fc61a4013a3f54a4be (#16352) Co-authored-by: Andrew Stucki --- agent/consul/gateways/controller_gateways.go | 92 ++++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/agent/consul/gateways/controller_gateways.go b/agent/consul/gateways/controller_gateways.go index f06b48581fa0..b1197d0ba4a5 100644 --- a/agent/consul/gateways/controller_gateways.go +++ b/agent/consul/gateways/controller_gateways.go @@ -71,7 +71,7 @@ func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Req func reconcileEntry[T structs.ControlledConfigEntry](store *state.Store, logger hclog.Logger, ctx context.Context, req controller.Request, reconciler func(ctx context.Context, req controller.Request, store *state.Store, entry T) error, cleaner func(ctx context.Context, req controller.Request, store *state.Store) error) error { _, entry, err := store.ConfigEntry(nil, req.Kind, req.Name, req.Meta) if err != nil { - requestLogger(logger, req).Error("error fetching config entry for reconciliation request", "error", err) + requestLogger(logger, req).Warn("error fetching config entry for reconciliation request", "error", err) return err } @@ -87,12 +87,12 @@ func reconcileEntry[T structs.ControlledConfigEntry](store *state.Store, logger func (r *apiGatewayReconciler) enqueueCertificateReferencedGateways(store *state.Store, _ context.Context, req controller.Request) error { logger := certificateRequestLogger(r.logger, req) - logger.Debug("certificate changed, enqueueing dependent gateways") - defer logger.Debug("finished enqueuing gateways") + logger.Trace("certificate changed, enqueueing dependent gateways") + defer logger.Trace("finished enqueuing gateways") _, entries, err := store.ConfigEntriesByKind(nil, structs.APIGateway, acl.WildcardEnterpriseMeta()) if err != nil { - logger.Error("error retrieving api gateways", "error", err) + logger.Warn("error retrieving api gateways", "error", err) return err } @@ -127,12 +127,12 @@ func (r *apiGatewayReconciler) enqueueCertificateReferencedGateways(store *state func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("cleaning up bound gateway") - defer logger.Debug("finished cleaning up bound gateway") + logger.Trace("cleaning up bound gateway") + defer logger.Trace("finished cleaning up bound gateway") routes, err := retrieveAllRoutesFromStore(store) if err != nil { - logger.Error("error retrieving routes", "error", err) + logger.Warn("error retrieving routes", "error", err) return err } @@ -141,9 +141,9 @@ func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req contro for _, modifiedRoute := range removeGateway(resource, routes...) { routeLogger := routeLogger(logger, modifiedRoute) - routeLogger.Debug("persisting route status") + routeLogger.Trace("persisting route status") if err := r.updater.Update(modifiedRoute); err != nil { - routeLogger.Error("error removing gateway from route", "error", err) + routeLogger.Warn("error removing gateway from route", "error", err) return err } } @@ -156,20 +156,20 @@ func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req contro func (r *apiGatewayReconciler) reconcileBoundGateway(_ context.Context, req controller.Request, store *state.Store, bound *structs.BoundAPIGatewayConfigEntry) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("reconciling bound gateway") - defer logger.Debug("finished reconciling bound gateway") + logger.Trace("reconciling bound gateway") + defer logger.Trace("finished reconciling bound gateway") _, gateway, err := store.ConfigEntry(nil, structs.APIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving api gateway", "error", err) + logger.Warn("error retrieving api gateway", "error", err) return err } if gateway == nil { // delete the bound gateway - logger.Debug("deleting bound api gateway") + logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { - logger.Error("error deleting bound api gateway", "error", err) + logger.Warn("error deleting bound api gateway", "error", err) return err } } @@ -183,18 +183,18 @@ func (r *apiGatewayReconciler) reconcileBoundGateway(_ context.Context, req cont func (r *apiGatewayReconciler) cleanupGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("cleaning up deleted gateway") - defer logger.Debug("finished cleaning up deleted gateway") + logger.Trace("cleaning up deleted gateway") + defer logger.Trace("finished cleaning up deleted gateway") _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving bound api gateway", "error", err) + logger.Warn("error retrieving bound api gateway", "error", err) return err } - logger.Debug("deleting bound api gateway") + logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { - logger.Error("error deleting bound api gateway", "error", err) + logger.Warn("error deleting bound api gateway", "error", err) return err } @@ -212,8 +212,8 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle logger := gatewayRequestLogger(r.logger, req) - logger.Debug("started reconciling gateway") - defer logger.Debug("finished reconciling gateway") + logger.Trace("started reconciling gateway") + defer logger.Trace("finished reconciling gateway") updater := structs.NewStatusUpdater(gateway) // we clear out the initial status conditions since we're doing a full update @@ -222,21 +222,21 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle routes, err := retrieveAllRoutesFromStore(store) if err != nil { - logger.Error("error retrieving routes", "error", err) + logger.Warn("error retrieving routes", "error", err) return err } // construct the tuple we'll be working on to update state _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving bound api gateway", "error", err) + logger.Warn("error retrieving bound api gateway", "error", err) return err } meta := newGatewayMeta(gateway, bound) certificateErrors, err := meta.checkCertificates(store) if err != nil { - logger.Error("error checking gateway certificates", "error", err) + logger.Warn("error checking gateway certificates", "error", err) return err } @@ -286,9 +286,9 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle // now check if we need to update the gateway status if modifiedGateway, shouldUpdate := updater.UpdateEntry(); shouldUpdate { - logger.Debug("persisting gateway status") + logger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { - logger.Error("error persisting gateway status", "error", err) + logger.Warn("error persisting gateway status", "error", err) return err } } @@ -296,18 +296,18 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle // next update route statuses for _, modifiedRoute := range updatedRoutes { routeLogger := routeLogger(logger, modifiedRoute) - routeLogger.Debug("persisting route status") + routeLogger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { - routeLogger.Error("error persisting route status", "error", err) + routeLogger.Warn("error persisting route status", "error", err) return err } } // now update the bound state if it changed if bound == nil || !bound.(*structs.BoundAPIGatewayConfigEntry).IsSame(meta.BoundGateway) { - logger.Debug("persisting bound api gateway") + logger.Trace("persisting bound api gateway") if err := r.updater.Update(meta.BoundGateway); err != nil { - logger.Error("error persisting bound api gateway", "error", err) + logger.Warn("error persisting bound api gateway", "error", err) return err } } @@ -320,20 +320,20 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle func (r *apiGatewayReconciler) cleanupRoute(_ context.Context, req controller.Request, store *state.Store) error { logger := routeRequestLogger(r.logger, req) - logger.Debug("cleaning up route") - defer logger.Debug("finished cleaning up route") + logger.Trace("cleaning up route") + defer logger.Trace("finished cleaning up route") meta, err := getAllGatewayMeta(store) if err != nil { - logger.Error("error retrieving gateways", "error", err) + logger.Warn("error retrieving gateways", "error", err) return err } for _, modifiedGateway := range removeRoute(requestToResourceRef(req), meta...) { gatewayLogger := gatewayLogger(logger, modifiedGateway.BoundGateway) - gatewayLogger.Debug("persisting bound gateway state") + gatewayLogger.Trace("persisting bound gateway state") if err := r.updater.Update(modifiedGateway.BoundGateway); err != nil { - gatewayLogger.Error("error updating bound api gateway", "error", err) + gatewayLogger.Warn("error updating bound api gateway", "error", err) return err } } @@ -355,12 +355,12 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. logger := routeRequestLogger(r.logger, req) - logger.Debug("reconciling route") - defer logger.Debug("finished reconciling route") + logger.Trace("reconciling route") + defer logger.Trace("finished reconciling route") meta, err := getAllGatewayMeta(store) if err != nil { - logger.Error("error retrieving gateways", "error", err) + logger.Warn("error retrieving gateways", "error", err) return err } @@ -378,9 +378,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. modifiedGateway, shouldUpdate := gateway.checkConflicts() if shouldUpdate { gatewayLogger := gatewayLogger(logger, modifiedGateway) - gatewayLogger.Debug("persisting gateway status") + gatewayLogger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { - gatewayLogger.Error("error persisting gateway", "error", err) + gatewayLogger.Warn("error persisting gateway", "error", err) return err } } @@ -388,9 +388,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. // next update the route status if modifiedRoute, shouldUpdate := updater.UpdateEntry(); shouldUpdate { - r.logger.Debug("persisting route status") + r.logger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { - r.logger.Error("error persisting route", "error", err) + r.logger.Warn("error persisting route", "error", err) return err } } @@ -398,9 +398,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. // now update all of the bound gateways that have been modified for _, bound := range modifiedGateways { gatewayLogger := gatewayLogger(logger, bound) - gatewayLogger.Debug("persisting bound api gateway") + gatewayLogger.Trace("persisting bound api gateway") if err := r.updater.Update(bound); err != nil { - gatewayLogger.Error("error persisting bound api gateway", "error", err) + gatewayLogger.Warn("error persisting bound api gateway", "error", err) return err } } @@ -413,7 +413,7 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. for _, service := range route.GetServiceNames() { _, chainSet, err := store.ReadDiscoveryChainConfigEntries(ws, service.Name, pointerTo(service.EnterpriseMeta)) if err != nil { - logger.Error("error reading discovery chain", "error", err) + logger.Warn("error reading discovery chain", "error", err) return err } @@ -481,7 +481,7 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. } // the route is valid, attempt to bind it to all gateways - r.logger.Debug("binding routes to gateway") + r.logger.Trace("binding routes to gateway") modifiedGateways, boundRefs, bindErrors := bindRoutesToGateways(route, meta...) // set the status of the references that are bound From 3d232666ae06f0f5da749f9bc99915eac342fb19 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 23:34:39 -0500 Subject: [PATCH 021/421] Backport of [API Gateway] Fix targeting service splitters in HTTPRoutes into release/1.15.x (#16353) * backport of commit 06fc59c26f362c8630d12e0833506580a4837648 * backport of commit cd4e6e9677d4ed4ea25e3251c072db72181c4bd2 --------- Co-authored-by: Andrew Stucki --- agent/consul/discoverychain/gateway.go | 39 +++ agent/consul/discoverychain/gateway_test.go | 254 ++++++++++++++++++ agent/structs/config_entry_routes.go | 8 +- .../capture.sh | 3 + .../service_gateway.hcl | 4 + .../service_s3.hcl | 9 + .../setup.sh | 80 ++++++ .../vars.sh | 3 + .../verify.bats | 23 ++ 9 files changed, 419 insertions(+), 4 deletions(-) create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/capture.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_gateway.hcl create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_s3.hcl create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/setup.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/vars.sh create mode 100644 test/integration/connect/envoy/case-api-gateway-http-splitter-targets/verify.bats diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index c9acdbbfda2f..35a8992d80f1 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -5,6 +5,7 @@ import ( "hash/crc32" "sort" "strconv" + "strings" "github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/structs" @@ -126,6 +127,23 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover if err != nil { return nil, nil, err } + + // fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present + for name, node := range compiled.Nodes { + switch node.Type { + // we should only have these two types + case structs.DiscoveryGraphNodeTypeRouter: + for i, route := range node.Routes { + node.Routes[i].NextNode = targetForResolverNode(route.NextNode, chains) + } + case structs.DiscoveryGraphNodeTypeSplitter: + for i, split := range node.Splits { + node.Splits[i].NextNode = targetForResolverNode(split.NextNode, chains) + } + } + compiled.Nodes[name] = node + } + for _, c := range chains { for id, target := range c.Targets { compiled.Targets[id] = target @@ -177,6 +195,27 @@ func (l *GatewayChainSynthesizer) consolidateHTTPRoutes() []structs.HTTPRouteCon return routes } +func targetForResolverNode(nodeName string, chains []*structs.CompiledDiscoveryChain) string { + resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + splitterPrefix := structs.DiscoveryGraphNodeTypeSplitter + ":" + + if !strings.HasPrefix(nodeName, resolverPrefix) { + return nodeName + } + + splitterName := splitterPrefix + strings.TrimPrefix(nodeName, resolverPrefix) + + for _, c := range chains { + for name, node := range c.Nodes { + if node.IsSplitter() && strings.HasPrefix(splitterName, name) { + return name + } + } + } + + return nodeName +} + func hostsKey(hosts ...string) string { sort.Strings(hosts) hostsHash := crc32.NewIEEE() diff --git a/agent/consul/discoverychain/gateway_test.go b/agent/consul/discoverychain/gateway_test.go index 1c44f680b7c0..71e66b051275 100644 --- a/agent/consul/discoverychain/gateway_test.go +++ b/agent/consul/discoverychain/gateway_test.go @@ -3,6 +3,7 @@ package discoverychain import ( "testing" + "github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/structs" "github.com/stretchr/testify/require" ) @@ -640,3 +641,256 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }) } } + +func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + synthesizer *GatewayChainSynthesizer + route *structs.HTTPRouteConfigEntry + entries []structs.ConfigEntry + expectedDiscoveryChain *structs.CompiledDiscoveryChain + }{ + "HTTP-Route with nested splitters": { + synthesizer: NewGatewayChainSynthesizer("dc1", "domain", "suffix", &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + }), + route: &structs.HTTPRouteConfigEntry{ + Kind: structs.HTTPRoute, + Name: "test", + Rules: []structs.HTTPRouteRule{{ + Services: []structs.HTTPService{{ + Name: "splitter-one", + }}, + }}, + }, + entries: []structs.ConfigEntry{ + &structs.ServiceSplitterConfigEntry{ + Kind: structs.ServiceSplitter, + Name: "splitter-one", + Splits: []structs.ServiceSplit{{ + Service: "service-one", + Weight: 50, + }, { + Service: "splitter-two", + Weight: 50, + }}, + }, + &structs.ServiceSplitterConfigEntry{ + Kind: structs.ServiceSplitter, + Name: "splitter-two", + Splits: []structs.ServiceSplit{{ + Service: "service-two", + Weight: 50, + }, { + Service: "service-three", + Weight: 50, + }}, + }, + &structs.ProxyConfigEntry{ + Kind: structs.ProxyConfigGlobal, + Name: "global", + Config: map[string]interface{}{ + "protocol": "http", + }, + }, + }, + expectedDiscoveryChain: &structs.CompiledDiscoveryChain{ + ServiceName: "gateway-suffix-9b9265b", + Namespace: "default", + Partition: "default", + Datacenter: "dc1", + Protocol: "http", + StartNode: "router:gateway-suffix-9b9265b.default.default", + Nodes: map[string]*structs.DiscoveryGraphNode{ + "resolver:gateway-suffix-9b9265b.default.default.dc1": { + Type: "resolver", + Name: "gateway-suffix-9b9265b.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "gateway-suffix-9b9265b.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:service-one.default.default.dc1": { + Type: "resolver", + Name: "service-one.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-one.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:service-three.default.default.dc1": { + Type: "resolver", + Name: "service-three.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-three.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:service-two.default.default.dc1": { + Type: "resolver", + Name: "service-two.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-two.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:splitter-one.default.default.dc1": { + Type: "resolver", + Name: "splitter-one.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "splitter-one.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "router:gateway-suffix-9b9265b.default.default": { + Type: "router", + Name: "gateway-suffix-9b9265b.default.default", + Routes: []*structs.DiscoveryRoute{{ + Definition: &structs.ServiceRoute{ + Match: &structs.ServiceRouteMatch{ + HTTP: &structs.ServiceRouteHTTPMatch{ + PathPrefix: "/", + }, + }, + Destination: &structs.ServiceRouteDestination{ + Service: "splitter-one", + Partition: "default", + Namespace: "default", + RequestHeaders: &structs.HTTPHeaderModifiers{ + Add: make(map[string]string), + Set: make(map[string]string), + }, + }, + }, + NextNode: "splitter:splitter-one.default.default", + }, { + Definition: &structs.ServiceRoute{ + Match: &structs.ServiceRouteMatch{ + HTTP: &structs.ServiceRouteHTTPMatch{ + PathPrefix: "/", + }, + }, + Destination: &structs.ServiceRouteDestination{ + Service: "gateway-suffix-9b9265b", + Partition: "default", + Namespace: "default", + }, + }, + NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", + }}, + }, + "splitter:splitter-one.default.default": { + Type: structs.DiscoveryGraphNodeTypeSplitter, + Name: "splitter-one.default.default", + Splits: []*structs.DiscoverySplit{{ + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-one", + }, + Weight: 50, + NextNode: "resolver:service-one.default.default.dc1", + }, { + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-two", + }, + Weight: 25, + NextNode: "resolver:service-two.default.default.dc1", + }, { + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-three", + }, + Weight: 25, + NextNode: "resolver:service-three.default.default.dc1", + }}, + }, + }, Targets: map[string]*structs.DiscoveryTarget{ + "gateway-suffix-9b9265b.default.default.dc1": { + ID: "gateway-suffix-9b9265b.default.default.dc1", + Service: "gateway-suffix-9b9265b", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "gateway-suffix-9b9265b.default.dc1.internal.domain", + Name: "gateway-suffix-9b9265b.default.dc1.internal.domain", + }, + "service-one.default.default.dc1": { + ID: "service-one.default.default.dc1", + Service: "service-one", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-one.default.dc1.internal.domain", + Name: "service-one.default.dc1.internal.domain", + }, + "service-three.default.default.dc1": { + ID: "service-three.default.default.dc1", + Service: "service-three", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-three.default.dc1.internal.domain", + Name: "service-three.default.dc1.internal.domain", + }, + "service-two.default.default.dc1": { + ID: "service-two.default.default.dc1", + Service: "service-two", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-two.default.dc1.internal.domain", + Name: "service-two.default.dc1.internal.domain", + }, + "splitter-one.default.default.dc1": { + ID: "splitter-one.default.default.dc1", + Service: "splitter-one", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "splitter-one.default.dc1.internal.domain", + Name: "splitter-one.default.dc1.internal.domain", + }, + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + service := tc.entries[0] + entries := configentry.NewDiscoveryChainSet() + entries.AddEntries(tc.entries...) + compiled, err := Compile(CompileRequest{ + ServiceName: service.GetName(), + EvaluateInNamespace: service.GetEnterpriseMeta().NamespaceOrDefault(), + EvaluateInPartition: service.GetEnterpriseMeta().PartitionOrDefault(), + EvaluateInDatacenter: "dc1", + EvaluateInTrustDomain: "domain", + Entries: entries, + }) + require.NoError(t, err) + + tc.synthesizer.SetHostname("*") + tc.synthesizer.AddHTTPRoute(*tc.route) + + chains := []*structs.CompiledDiscoveryChain{compiled} + _, discoveryChains, err := tc.synthesizer.Synthesize(chains...) + + require.NoError(t, err) + require.Len(t, discoveryChains, 1) + require.Equal(t, tc.expectedDiscoveryChain, discoveryChains[0]) + }) + } +} diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index fa4427776a40..0571a273f027 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -79,9 +79,9 @@ func (e *HTTPRouteConfigEntry) Normalize() error { for i, parent := range e.Parents { if parent.Kind == "" { parent.Kind = APIGateway - parent.EnterpriseMeta.Normalize() - e.Parents[i] = parent } + parent.EnterpriseMeta.Normalize() + e.Parents[i] = parent } for i, rule := range e.Rules { @@ -505,9 +505,9 @@ func (e *TCPRouteConfigEntry) Normalize() error { for i, parent := range e.Parents { if parent.Kind == "" { parent.Kind = APIGateway - parent.EnterpriseMeta.Normalize() - e.Parents[i] = parent } + parent.EnterpriseMeta.Normalize() + e.Parents[i] = parent } for i, service := range e.Services { diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/capture.sh b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/capture.sh new file mode 100644 index 000000000000..8ba0e0ddabc6 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/capture.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 api-gateway primary || true \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_gateway.hcl b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_gateway.hcl new file mode 100644 index 000000000000..486c25c59e5f --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_gateway.hcl @@ -0,0 +1,4 @@ +services { + name = "api-gateway" + kind = "api-gateway" +} diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_s3.hcl b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_s3.hcl new file mode 100644 index 000000000000..2f6d05e0feb0 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/service_s3.hcl @@ -0,0 +1,9 @@ +services { + id = "s3" + name = "s3" + port = 8182 + + connect { + sidecar_service {} + } +} diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/setup.sh b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/setup.sh new file mode 100644 index 000000000000..47916da173c1 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/setup.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +set -euo pipefail + +upsert_config_entry primary ' +kind = "api-gateway" +name = "api-gateway" +listeners = [ + { + name = "listener-one" + port = 9999 + protocol = "http" + } +] +' + +upsert_config_entry primary ' +Kind = "proxy-defaults" +Name = "global" +Config { + protocol = "http" +} +' + +upsert_config_entry primary ' +kind = "http-route" +name = "api-gateway-route-one" +rules = [ + { + services = [ + { + name = "splitter-one" + } + ] + } +] +parents = [ + { + name = "api-gateway" + sectionName = "listener-one" + } +] +' + +upsert_config_entry primary ' +kind = "service-splitter" +name = "splitter-one" +splits = [ + { + weight = 50, + service = "s1" + }, + { + weight = 50, + service = "splitter-two" + }, +] +' + +upsert_config_entry primary ' +kind = "service-splitter" +name = "splitter-two" +splits = [ + { + weight = 50, + service = "s2" + }, + { + weight = 50, + service = "s3" + }, +] +' + +register_services primary + +gen_envoy_bootstrap api-gateway 20000 primary true +gen_envoy_bootstrap s1 19000 +gen_envoy_bootstrap s2 19001 +gen_envoy_bootstrap s3 19002 \ No newline at end of file diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/vars.sh b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/vars.sh new file mode 100644 index 000000000000..38a47d852783 --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/vars.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +export REQUIRED_SERVICES="$DEFAULT_REQUIRED_SERVICES api-gateway-primary" diff --git a/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/verify.bats b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/verify.bats new file mode 100644 index 000000000000..50d447516b1d --- /dev/null +++ b/test/integration/connect/envoy/case-api-gateway-http-splitter-targets/verify.bats @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +load helpers + +@test "api gateway proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "api gateway should be accepted and not conflicted" { + assert_config_entry_status Accepted True Accepted primary api-gateway api-gateway + assert_config_entry_status Conflicted False NoConflict primary api-gateway api-gateway +} + +@test "api gateway should have healthy endpoints for s1" { + assert_config_entry_status Bound True Bound primary http-route api-gateway-route-one + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 +} + +@test "api gateway should be able to connect to s1, s2, and s3 via configured port" { + run retry_default assert_expected_fortio_name_pattern ^FORTIO_NAME=s1$ + run retry_default assert_expected_fortio_name_pattern ^FORTIO_NAME=s2$ + run retry_default assert_expected_fortio_name_pattern ^FORTIO_NAME=s3$ +} \ No newline at end of file From fca4b563d4c2974b6373fd32b1e629dd8f5fa41f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Feb 2023 23:36:25 -0500 Subject: [PATCH 022/421] Backport of [API Gateway] Various fixes for Config Entry fields into release/1.15.x (#16354) * [API Gateway] Various fixes for Config Entry fields * simplify logic per PR review --------- Co-authored-by: Andrew Stucki --- .../discoverychain/gateway_httproute.go | 17 +- agent/structs/config_entry_routes.go | 15 +- agent/structs/structs.deepcopy.go | 12 +- api/config_entry_routes.go | 7 +- proto/pbconfigentry/config_entry.gen.go | 26 +- proto/pbconfigentry/config_entry.pb.go | 363 +++++++++--------- proto/pbconfigentry/config_entry.proto | 6 +- 7 files changed, 206 insertions(+), 240 deletions(-) diff --git a/agent/consul/discoverychain/gateway_httproute.go b/agent/consul/discoverychain/gateway_httproute.go index aaaec12e6b19..30968c9a4d5d 100644 --- a/agent/consul/discoverychain/gateway_httproute.go +++ b/agent/consul/discoverychain/gateway_httproute.go @@ -77,15 +77,14 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser for idx, rule := range route.Rules { modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers) - prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrites) + prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrite) var destination structs.ServiceRouteDestination if len(rule.Services) == 1 { - // TODO open question: is there a use case where someone might want to set the rewrite to ""? service := rule.Services[0] - servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrites) - if servicePrefixRewrite == "" { + servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrite) + if service.Filters.URLRewrite == nil { servicePrefixRewrite = prefixRewrite } serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers) @@ -176,13 +175,11 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser return router, splitters, defaults } -func httpRouteFiltersToDestinationPrefixRewrite(rewrites []structs.URLRewrite) string { - for _, rewrite := range rewrites { - if rewrite.Path != "" { - return rewrite.Path - } +func httpRouteFiltersToDestinationPrefixRewrite(rewrite *structs.URLRewrite) string { + if rewrite == nil { + return "" } - return "" + return rewrite.Path } // httpRouteFiltersToServiceRouteHeaderModifier will consolidate a list of HTTP filters diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 0571a273f027..801e22f18cbe 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -211,16 +211,14 @@ func validateFilters(filter HTTPFilters) error { } } - for i, rewrite := range filter.URLRewrites { - if err := validateURLRewrite(rewrite); err != nil { - return fmt.Errorf("HTTPFilters, URLRewrite[%d], %w", i, err) - } + if err := validateURLRewrite(filter.URLRewrite); err != nil { + return fmt.Errorf("HTTPFilters, URLRewrite, %w", err) } return nil } -func validateURLRewrite(rewrite URLRewrite) error { +func validateURLRewrite(rewrite *URLRewrite) error { // TODO: we don't really have validation of the actual params // passed as "PrefixRewrite" in our discoverychain config // entries, figure out if we should have something here @@ -403,8 +401,8 @@ type HTTPQueryMatch struct { // HTTPFilters specifies a list of filters used to modify a request // before it is routed to an upstream. type HTTPFilters struct { - Headers []HTTPHeaderFilter - URLRewrites []URLRewrite + Headers []HTTPHeaderFilter + URLRewrite *URLRewrite } // HTTPHeaderFilter specifies how HTTP headers should be modified. @@ -554,9 +552,6 @@ func (e *TCPRouteConfigEntry) CanWrite(authz acl.Authorizer) error { // TCPService is a service reference for a TCPRoute type TCPService struct { Name string - // Weight specifies the proportion of requests forwarded to the referenced service. - // This is computed as weight/(sum of all weights in the list of services). - Weight int acl.EnterpriseMeta } diff --git a/agent/structs/structs.deepcopy.go b/agent/structs/structs.deepcopy.go index d52585771e7c..43f94674c5f5 100644 --- a/agent/structs/structs.deepcopy.go +++ b/agent/structs/structs.deepcopy.go @@ -329,9 +329,9 @@ func (o *HTTPRouteConfigEntry) DeepCopy() *HTTPRouteConfigEntry { } } } - if o.Rules[i2].Filters.URLRewrites != nil { - cp.Rules[i2].Filters.URLRewrites = make([]URLRewrite, len(o.Rules[i2].Filters.URLRewrites)) - copy(cp.Rules[i2].Filters.URLRewrites, o.Rules[i2].Filters.URLRewrites) + if o.Rules[i2].Filters.URLRewrite != nil { + cp.Rules[i2].Filters.URLRewrite = new(URLRewrite) + *cp.Rules[i2].Filters.URLRewrite = *o.Rules[i2].Filters.URLRewrite } if o.Rules[i2].Matches != nil { cp.Rules[i2].Matches = make([]HTTPMatch, len(o.Rules[i2].Matches)) @@ -373,9 +373,9 @@ func (o *HTTPRouteConfigEntry) DeepCopy() *HTTPRouteConfigEntry { } } } - if o.Rules[i2].Services[i4].Filters.URLRewrites != nil { - cp.Rules[i2].Services[i4].Filters.URLRewrites = make([]URLRewrite, len(o.Rules[i2].Services[i4].Filters.URLRewrites)) - copy(cp.Rules[i2].Services[i4].Filters.URLRewrites, o.Rules[i2].Services[i4].Filters.URLRewrites) + if o.Rules[i2].Services[i4].Filters.URLRewrite != nil { + cp.Rules[i2].Services[i4].Filters.URLRewrite = new(URLRewrite) + *cp.Rules[i2].Services[i4].Filters.URLRewrite = *o.Rules[i2].Services[i4].Filters.URLRewrite } } } diff --git a/api/config_entry_routes.go b/api/config_entry_routes.go index 8561c02eaf7a..2edf9b23a4c0 100644 --- a/api/config_entry_routes.go +++ b/api/config_entry_routes.go @@ -49,9 +49,6 @@ func (a *TCPRouteConfigEntry) GetModifyIndex() uint64 { return a.ModifyIndex // TCPService is a service reference for a TCPRoute type TCPService struct { Name string - // Weight specifies the proportion of requests forwarded to the referenced service. - // This is computed as weight/(sum of all weights in the list of services). - Weight int // Partition is the partition the config entry is associated with. // Partitioning is a Consul Enterprise feature. @@ -195,8 +192,8 @@ type HTTPQueryMatch struct { // HTTPFilters specifies a list of filters used to modify a request // before it is routed to an upstream. type HTTPFilters struct { - Headers []HTTPHeaderFilter - URLRewrites []URLRewrite + Headers []HTTPHeaderFilter + URLRewrite *URLRewrite } // HTTPHeaderFilter specifies how HTTP headers should be modified. diff --git a/proto/pbconfigentry/config_entry.gen.go b/proto/pbconfigentry/config_entry.gen.go index b5ae38610921..d4f2404b1f27 100644 --- a/proto/pbconfigentry/config_entry.gen.go +++ b/proto/pbconfigentry/config_entry.gen.go @@ -364,13 +364,10 @@ func HTTPFiltersToStructs(s *HTTPFilters, t *structs.HTTPFilters) { } } } - { - t.URLRewrites = make([]structs.URLRewrite, len(s.URLRewrites)) - for i := range s.URLRewrites { - if s.URLRewrites[i] != nil { - URLRewriteToStructs(s.URLRewrites[i], &t.URLRewrites[i]) - } - } + if s.URLRewrite != nil { + var x structs.URLRewrite + URLRewriteToStructs(s.URLRewrite, &x) + t.URLRewrite = &x } } func HTTPFiltersFromStructs(t *structs.HTTPFilters, s *HTTPFilters) { @@ -387,15 +384,10 @@ func HTTPFiltersFromStructs(t *structs.HTTPFilters, s *HTTPFilters) { } } } - { - s.URLRewrites = make([]*URLRewrite, len(t.URLRewrites)) - for i := range t.URLRewrites { - { - var x URLRewrite - URLRewriteFromStructs(&t.URLRewrites[i], &x) - s.URLRewrites[i] = &x - } - } + if t.URLRewrite != nil { + var x URLRewrite + URLRewriteFromStructs(t.URLRewrite, &x) + s.URLRewrite = &x } } func HTTPHeaderFilterToStructs(s *HTTPHeaderFilter, t *structs.HTTPHeaderFilter) { @@ -1635,7 +1627,6 @@ func TCPServiceToStructs(s *TCPService, t *structs.TCPService) { return } t.Name = s.Name - t.Weight = int(s.Weight) t.EnterpriseMeta = enterpriseMetaToStructs(s.EnterpriseMeta) } func TCPServiceFromStructs(t *structs.TCPService, s *TCPService) { @@ -1643,7 +1634,6 @@ func TCPServiceFromStructs(t *structs.TCPService, s *TCPService) { return } s.Name = t.Name - s.Weight = int32(t.Weight) s.EnterpriseMeta = enterpriseMetaFromStructs(t.EnterpriseMeta) } func TransparentProxyConfigToStructs(s *TransparentProxyConfig, t *structs.TransparentProxyConfig) { diff --git a/proto/pbconfigentry/config_entry.pb.go b/proto/pbconfigentry/config_entry.pb.go index 4d9d292d7d44..c528cd36feee 100644 --- a/proto/pbconfigentry/config_entry.pb.go +++ b/proto/pbconfigentry/config_entry.pb.go @@ -4916,8 +4916,8 @@ type HTTPFilters struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Headers []*HTTPHeaderFilter `protobuf:"bytes,1,rep,name=Headers,proto3" json:"Headers,omitempty"` - URLRewrites []*URLRewrite `protobuf:"bytes,2,rep,name=URLRewrites,proto3" json:"URLRewrites,omitempty"` + Headers []*HTTPHeaderFilter `protobuf:"bytes,1,rep,name=Headers,proto3" json:"Headers,omitempty"` + URLRewrite *URLRewrite `protobuf:"bytes,2,opt,name=URLRewrite,proto3" json:"URLRewrite,omitempty"` } func (x *HTTPFilters) Reset() { @@ -4959,9 +4959,9 @@ func (x *HTTPFilters) GetHeaders() []*HTTPHeaderFilter { return nil } -func (x *HTTPFilters) GetURLRewrites() []*URLRewrite { +func (x *HTTPFilters) GetURLRewrite() *URLRewrite { if x != nil { - return x.URLRewrites + return x.URLRewrite } return nil } @@ -5252,10 +5252,8 @@ type TCPService struct { unknownFields protoimpl.UnknownFields Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - // mog: func-to=int func-from=int32 - Weight int32 `protobuf:"varint,2,opt,name=Weight,proto3" json:"Weight,omitempty"` // mog: func-to=enterpriseMetaToStructs func-from=enterpriseMetaFromStructs - EnterpriseMeta *pbcommon.EnterpriseMeta `protobuf:"bytes,4,opt,name=EnterpriseMeta,proto3" json:"EnterpriseMeta,omitempty"` + EnterpriseMeta *pbcommon.EnterpriseMeta `protobuf:"bytes,2,opt,name=EnterpriseMeta,proto3" json:"EnterpriseMeta,omitempty"` } func (x *TCPService) Reset() { @@ -5297,13 +5295,6 @@ func (x *TCPService) GetName() string { return "" } -func (x *TCPService) GetWeight() int32 { - if x != nil { - return x.Weight - } - return 0 -} - func (x *TCPService) GetEnterpriseMeta() *pbcommon.EnterpriseMeta { if x != nil { return x.EnterpriseMeta @@ -6277,188 +6268,186 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb5, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x46, + 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb3, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x0b, 0x55, 0x52, 0x4c, - 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, - 0x65, 0x52, 0x0b, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x22, 0x20, - 0x0a, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, - 0x22, 0xc2, 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x12, 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x03, 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, - 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x0a, 0x55, 0x52, 0x4c, + 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, + 0x52, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, 0x20, 0x0a, 0x0a, + 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0xc2, + 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x12, + 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, + 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, 0x53, + 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, + 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, + 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, + 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, + 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfc, 0x02, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfc, 0x02, 0x0a, 0x08, 0x54, 0x43, - 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, + 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, + 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, + 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x2a, 0xfd, 0x01, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x4b, + 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, + 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x01, + 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, 0x6e, + 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, + 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, + 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x73, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0x06, + 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, 0x6f, 0x75, 0x6e, + 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, 0x12, 0x11, 0x0a, + 0x0d, 0x4b, 0x69, 0x6e, 0x64, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x09, + 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x10, 0x0a, 0x2a, 0x26, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, + 0x09, 0x0a, 0x05, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x2a, 0x50, 0x0a, + 0x09, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, + 0x12, 0x18, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x10, 0x02, 0x2a, + 0x7b, 0x0a, 0x0f, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, + 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x17, + 0x0a, 0x13, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, + 0x65, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x10, + 0x02, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, 0x4f, 0x0a, 0x1a, + 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x48, 0x54, + 0x54, 0x50, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, 0x2a, 0x92, 0x02, + 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, + 0x64, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x41, 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, + 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, + 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x47, 0x65, 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, 0x61, 0x64, 0x10, + 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, 0x12, 0x18, 0x0a, + 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, + 0x50, 0x61, 0x74, 0x63, 0x68, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, 0x74, 0x10, 0x07, + 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x50, 0x75, 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, 0x61, 0x63, 0x65, + 0x10, 0x09, 0x2a, 0xa7, 0x01, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, + 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, + 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, + 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x48, + 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, + 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, + 0x03, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, 0x68, 0x0a, 0x11, + 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, + 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x10, 0x01, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, + 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, + 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x01, + 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, - 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x0a, 0x54, 0x43, 0x50, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, - 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, - 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, - 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, - 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x2a, 0xfd, 0x01, - 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x4b, 0x69, 0x6e, 0x64, 0x55, 0x6e, - 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x4d, - 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4b, - 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, - 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x10, 0x05, - 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, - 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x4b, - 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x07, 0x12, - 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, 0x12, 0x11, 0x0a, 0x0d, 0x4b, 0x69, 0x6e, 0x64, - 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x4b, - 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x0a, 0x2a, 0x26, 0x0a, - 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x6c, - 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, - 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x2a, 0x50, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x78, - 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, - 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, - 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x10, 0x02, 0x2a, 0x7b, 0x0a, 0x0f, 0x4d, 0x65, - 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, - 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, - 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4d, 0x65, 0x73, - 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4e, 0x6f, 0x6e, 0x65, - 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, - 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, 0x4f, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x48, 0x54, 0x54, 0x50, 0x10, 0x00, 0x12, - 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, 0x2a, 0x92, 0x02, 0x0a, 0x0f, 0x48, 0x54, 0x54, - 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x12, - 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x41, - 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x10, 0x01, - 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, - 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x48, - 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x47, 0x65, - 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, 0x61, 0x64, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, - 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x61, 0x74, 0x63, 0x68, - 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, - 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, 0x74, 0x10, 0x07, 0x12, 0x16, 0x0a, 0x12, 0x48, - 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x75, - 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, 0x61, 0x63, 0x65, 0x10, 0x09, 0x2a, 0xa7, 0x01, - 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, - 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, - 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, - 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, - 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x53, - 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, 0x68, 0x0a, 0x11, 0x48, 0x54, 0x54, 0x50, 0x50, - 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, - 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, - 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, 0x22, 0x0a, - 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, - 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, - 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, - 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x48, - 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, - 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, - 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, 0x43, 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, - 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, 0x43, 0xaa, + 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xe2, + 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, + 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -6680,7 +6669,7 @@ var file_proto_pbconfigentry_config_entry_proto_depIdxs = []int32{ 8, // 105: hashicorp.consul.internal.configentry.HTTPPathMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatchType 9, // 106: hashicorp.consul.internal.configentry.HTTPQueryMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatchType 67, // 107: hashicorp.consul.internal.configentry.HTTPFilters.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter - 66, // 108: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrites:type_name -> hashicorp.consul.internal.configentry.URLRewrite + 66, // 108: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrite:type_name -> hashicorp.consul.internal.configentry.URLRewrite 86, // 109: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.AddEntry 87, // 110: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.SetEntry 65, // 111: hashicorp.consul.internal.configentry.HTTPService.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters diff --git a/proto/pbconfigentry/config_entry.proto b/proto/pbconfigentry/config_entry.proto index 6749d5838c5d..3fb6b649c1a0 100644 --- a/proto/pbconfigentry/config_entry.proto +++ b/proto/pbconfigentry/config_entry.proto @@ -806,7 +806,7 @@ message HTTPQueryMatch { // name=Structs message HTTPFilters { repeated HTTPHeaderFilter Headers = 1; - repeated URLRewrite URLRewrites = 2; + URLRewrite URLRewrite = 2; } // mog annotation: @@ -863,8 +863,6 @@ message TCPRoute { // name=Structs message TCPService { string Name = 1; - // mog: func-to=int func-from=int32 - int32 Weight = 2; // mog: func-to=enterpriseMetaToStructs func-from=enterpriseMetaFromStructs - common.EnterpriseMeta EnterpriseMeta = 4; + common.EnterpriseMeta EnterpriseMeta = 2; } From aad218d4d0b072e3e71982d7510a061f6e9a1c21 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 10:54:59 -0500 Subject: [PATCH 023/421] backport of commit bb33b6060bb2d1f8e7ab2b31530d4248928ffd10 (#16357) Co-authored-by: Xinyi Wang --- .../test/ratelimit/ratelimit_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/integration/consul-container/test/ratelimit/ratelimit_test.go b/test/integration/consul-container/test/ratelimit/ratelimit_test.go index bde1b44be9bf..cb5c259eefe3 100644 --- a/test/integration/consul-container/test/ratelimit/ratelimit_test.go +++ b/test/integration/consul-container/test/ratelimit/ratelimit_test.go @@ -46,6 +46,7 @@ func TestServerRequestRateLimit(t *testing.T) { description string cmd string operations []operation + mode string } getKV := action{ @@ -70,6 +71,7 @@ func TestServerRequestRateLimit(t *testing.T) { { description: "HTTP & net/RPC / Mode: disabled - errors: no / exceeded logs: no / metrics: no", cmd: `-hcl=limits { request_limits { mode = "disabled" read_rate = 0 write_rate = 0 }}`, + mode: "disabled", operations: []operation{ { action: putKV, @@ -88,6 +90,7 @@ func TestServerRequestRateLimit(t *testing.T) { { description: "HTTP & net/RPC / Mode: permissive - errors: no / exceeded logs: yes / metrics: yes", cmd: `-hcl=limits { request_limits { mode = "permissive" read_rate = 0 write_rate = 0 }}`, + mode: "permissive", operations: []operation{ { action: putKV, @@ -106,6 +109,7 @@ func TestServerRequestRateLimit(t *testing.T) { { description: "HTTP & net/RPC / Mode: enforcing - errors: yes / exceeded logs: yes / metrics: yes", cmd: `-hcl=limits { request_limits { mode = "enforcing" read_rate = 0 write_rate = 0 }}`, + mode: "enforcing", operations: []operation{ { action: putKV, @@ -154,7 +158,7 @@ func TestServerRequestRateLimit(t *testing.T) { // require.NoError(t, err) if metricsInfo != nil && err == nil { if op.expectMetric { - checkForMetric(r, metricsInfo, op.action.rateLimitOperation, op.action.rateLimitType) + checkForMetric(r, metricsInfo, op.action.rateLimitOperation, op.action.rateLimitType, tc.mode) } } @@ -171,17 +175,17 @@ func TestServerRequestRateLimit(t *testing.T) { } } -func checkForMetric(t *retry.R, metricsInfo *api.MetricsInfo, operationName string, expectedLimitType string) { - const counterName = "rpc.rate_limit.exceeded" +func checkForMetric(t *retry.R, metricsInfo *api.MetricsInfo, operationName string, expectedLimitType string, expectedMode string) { + const counterName = "consul.rpc.rate_limit.exceeded" var counter api.SampledValue for _, c := range metricsInfo.Counters { - if counter.Name == counterName { + if c.Name == counterName { counter = c break } } - require.NotNilf(t, counter, "counter not found: %s", counterName) + require.NotEmptyf(t, counter.Name, "counter not found: %s", counterName) operation, ok := counter.Labels["op"] require.True(t, ok) @@ -193,9 +197,9 @@ func checkForMetric(t *retry.R, metricsInfo *api.MetricsInfo, operationName stri require.True(t, ok) if operation == operationName { - require.Equal(t, 2, counter.Count) + require.GreaterOrEqual(t, counter.Count, 1) require.Equal(t, expectedLimitType, limitType) - require.Equal(t, "disabled", mode) + require.Equal(t, expectedMode, mode) } } From 6b5d96744d09034c61b3894fa1a258588103056a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 11:16:22 -0500 Subject: [PATCH 024/421] backport of commit 813264e160ff6264cc5ad775a8321e943c5c5b32 (#16359) Co-authored-by: cskh --- .../consul-container/libs/assert/service.go | 28 ++- .../libs/cluster/container.go | 6 +- .../consul-container/libs/service/connect.go | 30 ++- .../consul-container/libs/service/examples.go | 4 + .../consul-container/libs/service/gateway.go | 4 + .../consul-container/libs/service/helpers.go | 4 +- .../consul-container/libs/service/service.go | 1 + .../test/upgrade/peering_http_test.go | 210 +++++++++++++++++- 8 files changed, 263 insertions(+), 24 deletions(-) diff --git a/test/integration/consul-container/libs/assert/service.go b/test/integration/consul-container/libs/assert/service.go index ba46821ffdcb..15a03be3b61a 100644 --- a/test/integration/consul-container/libs/assert/service.go +++ b/test/integration/consul-container/libs/assert/service.go @@ -49,9 +49,17 @@ func CatalogNodeExists(t *testing.T, c *api.Client, nodeName string) { }) } +func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) { + doHTTPServiceEchoes(t, ip, port, path, nil) +} + +func HTTPServiceEchoesResHeader(t *testing.T, ip string, port int, path string, expectedResHeader map[string]string) { + doHTTPServiceEchoes(t, ip, port, path, expectedResHeader) +} + // HTTPServiceEchoes verifies that a post to the given ip/port combination returns the data // in the response body. Optional path can be provided to differentiate requests. -func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) { +func doHTTPServiceEchoes(t *testing.T, ip string, port int, path string, expectedResHeader map[string]string) { const phrase = "hello" failer := func() *retry.Timer { @@ -82,6 +90,24 @@ func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) { if !strings.Contains(string(body), phrase) { r.Fatal("received an incorrect response ", string(body)) } + + for k, v := range expectedResHeader { + if headerValues, ok := res.Header[k]; !ok { + r.Fatal("expected header not found", k) + } else { + found := false + for _, value := range headerValues { + if value == v { + found = true + break + } + } + + if !found { + r.Fatalf("header %s value not match want %s got %s ", k, v, headerValues) + } + } + } }) } diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index c9ce7792b6d5..bd4416a35bf4 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -26,8 +26,9 @@ const bootLogLine = "Consul agent running" const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED" // Exposed ports info -const MaxEnvoyOnNode = 10 // the max number of Envoy sidecar can run along with the agent, base is 19000 -const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream +const MaxEnvoyOnNode = 10 // the max number of Envoy sidecar can run along with the agent, base is 19000 +const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream +const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams // consulContainerNode implements the Agent interface by running a Consul agent // in a container. @@ -530,6 +531,7 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR // Envoy upstream listener pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", ServiceUpstreamLocalBindPort)) + pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", ServiceUpstreamLocalBindPort2)) // Reserve the exposed ports for Envoy admin port, e.g., 19000 - 19009 basePort := 19000 diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index ac4907d4e583..5a1121b88600 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -14,7 +14,6 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -23,7 +22,7 @@ type ConnectContainer struct { ctx context.Context container testcontainers.Container ip string - appPort int + appPort []int externalAdminPort int internalAdminPort int mappedPublicPort int @@ -52,6 +51,10 @@ func (g ConnectContainer) Export(partition, peer string, client *api.Client) err } func (g ConnectContainer) GetAddr() (string, int) { + return g.ip, g.appPort[0] +} + +func (g ConnectContainer) GetAddrs() (string, []int) { return g.ip, g.appPort } @@ -132,7 +135,7 @@ func (g ConnectContainer) GetStatus() (string, error) { // node. The container exposes port serviceBindPort and envoy admin port // (19000) by mapping them onto host ports. The container's name has a prefix // combining datacenter and name. -func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID string, serviceBindPort int, node libcluster.Agent) (*ConnectContainer, error) { +func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID string, serviceBindPorts []int, node cluster.Agent) (*ConnectContainer, error) { nodeConfig := node.GetConfig() if nodeConfig.ScratchDir == "" { return nil, fmt.Errorf("node ScratchDir is required") @@ -202,11 +205,19 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID } var ( - appPortStr = strconv.Itoa(serviceBindPort) + appPortStrs []string adminPortStr = strconv.Itoa(internalAdminPort) ) - info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{appPortStr, adminPortStr}) + for _, port := range serviceBindPorts { + appPortStrs = append(appPortStrs, strconv.Itoa(port)) + } + + // expose the app ports and the envoy adminPortStr on the agent container + exposedPorts := make([]string, len(appPortStrs)) + copy(exposedPorts, appPortStrs) + exposedPorts = append(exposedPorts, adminPortStr) + info, err := cluster.LaunchContainerOnNode(ctx, node, req, exposedPorts) if err != nil { return nil, err } @@ -215,14 +226,17 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID ctx: ctx, container: info.Container, ip: info.IP, - appPort: info.MappedPorts[appPortStr].Int(), externalAdminPort: info.MappedPorts[adminPortStr].Int(), internalAdminPort: internalAdminPort, serviceName: sidecarServiceName, } - fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %d\n", - serviceID, out.appPort, serviceBindPort) + for _, port := range appPortStrs { + out.appPort = append(out.appPort, info.MappedPorts[port].Int()) + } + + fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %v\n", + serviceID, out.appPort, serviceBindPorts) fmt.Printf("NewConnectService sidecar: name %s, mapped admin port %d, admin port %d\n", sidecarServiceName, out.externalAdminPort, internalAdminPort) diff --git a/test/integration/consul-container/libs/service/examples.go b/test/integration/consul-container/libs/service/examples.go index 9d95f6e9099f..836c7ea5138d 100644 --- a/test/integration/consul-container/libs/service/examples.go +++ b/test/integration/consul-container/libs/service/examples.go @@ -64,6 +64,10 @@ func (g exampleContainer) GetAddr() (string, int) { return g.ip, g.httpPort } +func (g exampleContainer) GetAddrs() (string, []int) { + return "", nil +} + func (g exampleContainer) Restart() error { return fmt.Errorf("Restart Unimplemented by ConnectContainer") } diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 5fb3a36184b4..4aabdb918d4c 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -53,6 +53,10 @@ func (g gatewayContainer) GetAddr() (string, int) { return g.ip, g.port } +func (g gatewayContainer) GetAddrs() (string, []int) { + return "", nil +} + func (g gatewayContainer) GetLogs() (string, error) { rc, err := g.container.Logs(context.Background()) if err != nil { diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index 55ad94bb7f6b..8de49e45c20f 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -61,7 +61,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts _ = serverService.Terminate() }) - serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", serviceOpts.ID), serviceOpts.ID, serviceOpts.HTTPPort, node) // bindPort not used + serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", serviceOpts.ID), serviceOpts.ID, []int{serviceOpts.HTTPPort}, node) // bindPort not used if err != nil { return nil, nil, err } @@ -117,7 +117,7 @@ func CreateAndRegisterStaticClientSidecar( } // Create a service and proxy instance - clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, libcluster.ServiceUpstreamLocalBindPort, node) + clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, []int{libcluster.ServiceUpstreamLocalBindPort}, node) if err != nil { return nil, err } diff --git a/test/integration/consul-container/libs/service/service.go b/test/integration/consul-container/libs/service/service.go index 57a3539a6412..2e706bfccaf2 100644 --- a/test/integration/consul-container/libs/service/service.go +++ b/test/integration/consul-container/libs/service/service.go @@ -12,6 +12,7 @@ type Service interface { // Export a service to the peering cluster Export(partition, peer string, client *api.Client) error GetAddr() (string, int) + GetAddrs() (string, []int) // GetAdminAddr returns the external admin address GetAdminAddr() (string, int) GetLogs() (string, error) diff --git a/test/integration/consul-container/test/upgrade/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering_http_test.go index aec03a3edb41..61ac485d39d5 100644 --- a/test/integration/consul-container/test/upgrade/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering_http_test.go @@ -21,10 +21,15 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { t.Parallel() type testcase struct { - oldversion string - targetVersion string - name string - create func(*cluster.Cluster) (libservice.Service, error) + oldversion string + targetVersion string + name string + // create creates addtional resources in peered clusters depending on cases, e.g., static-client, + // static server, and config-entries. It returns the proxy services, an assertation function to + // be called to verify the resources. + create func(*cluster.Cluster, *cluster.Cluster) (libservice.Service, libservice.Service, func(), error) + // extraAssertion adds additional assertion function to the common resources across cases. + // common resources includes static-client in dialing cluster, and static-server in accepting cluster. extraAssertion func(int) } tcs := []testcase{ @@ -38,8 +43,8 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { oldversion: "1.14", targetVersion: utils.TargetVersion, name: "basic", - create: func(c *cluster.Cluster) (libservice.Service, error) { - return nil, nil + create: func(accepting *cluster.Cluster, dialing *cluster.Cluster) (libservice.Service, libservice.Service, func(), error) { + return nil, nil, func() {}, nil }, extraAssertion: func(clientUpstreamPort int) {}, }, @@ -49,7 +54,8 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { name: "http_router", // Create a second static-service at the client agent of accepting cluster and // a service-router that routes /static-server-2 to static-server-2 - create: func(c *cluster.Cluster) (libservice.Service, error) { + create: func(accepting *cluster.Cluster, dialing *cluster.Cluster) (libservice.Service, libservice.Service, func(), error) { + c := accepting serviceOpts := &libservice.ServiceOpts{ Name: libservice.StaticServer2ServiceName, ID: "static-server-2", @@ -60,7 +66,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(c.Clients()[0], serviceOpts) libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName) if err != nil { - return nil, err + return nil, nil, nil, err } err = c.ConfigEntryWrite(&api.ProxyConfigEntry{ Kind: api.ProxyDefaults, @@ -70,7 +76,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { }, }) if err != nil { - return nil, err + return nil, nil, nil, err } routerConfigEntry := &api.ServiceRouterConfigEntry{ Kind: api.ServiceRouter, @@ -90,12 +96,127 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { }, } err = c.ConfigEntryWrite(routerConfigEntry) - return serverConnectProxy, err + return serverConnectProxy, nil, func() {}, err }, extraAssertion: func(clientUpstreamPort int) { libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d/static-server-2", clientUpstreamPort), "static-server-2") }, }, + { + oldversion: "1.14", + targetVersion: utils.TargetVersion, + name: "http splitter and resolver", + // In addtional to the basic topology, this case provisions the following + // services in the dialing cluster: + // + // - a new static-client at server_0 that has two upstreams: split-static-server (5000) + // and peer-static-server (5001) + // - a local static-server service at client_0 + // - service-splitter named split-static-server w/ 2 services: "local-static-server" and + // "peer-static-server". + // - service-resolved named local-static-server + // - service-resolved named peer-static-server + create: func(accepting *cluster.Cluster, dialing *cluster.Cluster) (libservice.Service, libservice.Service, func(), error) { + err := dialing.ConfigEntryWrite(&api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: "global", + Config: map[string]interface{}{ + "protocol": "http", + }, + }) + if err != nil { + return nil, nil, nil, err + } + + clientConnectProxy, err := createAndRegisterStaticClientSidecarWithSplittingUpstreams(dialing) + if err != nil { + return nil, nil, nil, fmt.Errorf("error creating client connect proxy in cluster %s", dialing.NetworkName) + } + + // make a resolver for service peer-static-server + resolverConfigEntry := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: "peer-static-server", + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServerServiceName, + Peer: libtopology.DialingPeerName, + }, + } + err = dialing.ConfigEntryWrite(resolverConfigEntry) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing resolver config entry for %s", resolverConfigEntry.Name) + } + + // make a splitter for service split-static-server + splitter := &api.ServiceSplitterConfigEntry{ + Kind: api.ServiceSplitter, + Name: "split-static-server", + Splits: []api.ServiceSplit{ + { + Weight: 50, + Service: "local-static-server", + ResponseHeaders: &api.HTTPHeaderModifiers{ + Set: map[string]string{ + "x-test-split": "local", + }, + }, + }, + { + Weight: 50, + Service: "peer-static-server", + ResponseHeaders: &api.HTTPHeaderModifiers{ + Set: map[string]string{ + "x-test-split": "peer", + }, + }, + }, + }, + } + err = dialing.ConfigEntryWrite(splitter) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing splitter config entry for %s", splitter.Name) + } + + // make a resolver for service local-static-server + resolverConfigEntry = &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: "local-static-server", + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServerServiceName, + }, + } + err = dialing.ConfigEntryWrite(resolverConfigEntry) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing resolver config entry for %s", resolverConfigEntry.Name) + } + + // Make a static-server in dialing cluster + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8081, + GRPCPort: 8078, + } + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts) + libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName) + if err != nil { + return nil, nil, nil, err + } + + _, appPorts := clientConnectProxy.GetAddrs() + assertionFn := func() { + libassert.HTTPServiceEchoesResHeader(t, "localhost", appPorts[0], "", map[string]string{ + "X-Test-Split": "local", + }) + libassert.HTTPServiceEchoesResHeader(t, "localhost", appPorts[0], "", map[string]string{ + "X-Test-Split": "peer", + }) + libassert.HTTPServiceEchoes(t, "localhost", appPorts[0], "") + } + return serverConnectProxy, clientConnectProxy, assertionFn, nil + }, + extraAssertion: func(clientUpstreamPort int) {}, + }, } run := func(t *testing.T, tc testcase) { @@ -115,7 +236,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { _, staticClientPort := dialing.Container.GetAddr() _, appPort := dialing.Container.GetAddr() - _, err = tc.create(acceptingCluster) + _, secondClientProxy, assertionAdditionalResources, err := tc.create(acceptingCluster, dialingCluster) require.NoError(t, err) tc.extraAssertion(appPort) @@ -145,6 +266,12 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { require.NoError(t, accepting.Container.Restart()) libassert.HTTPServiceEchoes(t, "localhost", staticClientPort, "") + // restart the secondClientProxy if exist + if secondClientProxy != nil { + require.NoError(t, secondClientProxy.Restart()) + } + assertionAdditionalResources() + clientSidecarService, err := libservice.CreateAndRegisterStaticClientSidecar(dialingCluster.Servers()[0], libtopology.DialingPeerName, true) require.NoError(t, err) _, port := clientSidecarService.GetAddr() @@ -165,3 +292,64 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { // time.Sleep(3 * time.Second) } } + +// createAndRegisterStaticClientSidecarWithSplittingUpstreams creates a static-client-1 that +// has two upstreams: split-static-server (5000) and peer-static-server (5001) +func createAndRegisterStaticClientSidecarWithSplittingUpstreams(c *cluster.Cluster) (*libservice.ConnectContainer, error) { + // Do some trickery to ensure that partial completion is correctly torn + // down, but successful execution is not. + var deferClean utils.ResettableDefer + defer deferClean.Execute() + + node := c.Servers()[0] + mgwMode := api.MeshGatewayModeLocal + + // Register the static-client service and sidecar first to prevent race with sidecar + // trying to get xDS before it's ready + req := &api.AgentServiceRegistration{ + Name: libservice.StaticClientServiceName, + Port: 8080, + Connect: &api.AgentServiceConnect{ + SidecarService: &api.AgentServiceRegistration{ + Proxy: &api.AgentServiceConnectProxyConfig{ + Upstreams: []api.Upstream{ + { + DestinationName: "split-static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: cluster.ServiceUpstreamLocalBindPort, + MeshGateway: api.MeshGatewayConfig{ + Mode: mgwMode, + }, + }, + { + DestinationName: "peer-static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: cluster.ServiceUpstreamLocalBindPort2, + MeshGateway: api.MeshGatewayConfig{ + Mode: mgwMode, + }, + }, + }, + }, + }, + }, + } + + if err := node.GetClient().Agent().ServiceRegister(req); err != nil { + return nil, err + } + + // Create a service and proxy instance + clientConnectProxy, err := libservice.NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", libservice.StaticClientServiceName), libservice.StaticClientServiceName, []int{cluster.ServiceUpstreamLocalBindPort, cluster.ServiceUpstreamLocalBindPort2}, node) + if err != nil { + return nil, err + } + deferClean.Add(func() { + _ = clientConnectProxy.Terminate() + }) + + // disable cleanup functions now that we have an object with a Terminate() function + deferClean.Reset() + + return clientConnectProxy, nil +} From 87d97798bf60670a077f297d6ba589348c7e9678 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 11:51:54 -0500 Subject: [PATCH 025/421] backport of commit a0598b81a3ac334a767b5d143fd46465c1b64baf (#16363) Co-authored-by: Derek Menteer --- .changelog/16358.txt | 3 +++ Dockerfile | 4 ++-- test/integration/connect/envoy/Dockerfile-tcpdump | 4 ++-- test/integration/connect/envoy/helpers.bash | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 .changelog/16358.txt diff --git a/.changelog/16358.txt b/.changelog/16358.txt new file mode 100644 index 000000000000..91fcfe4505c4 --- /dev/null +++ b/.changelog/16358.txt @@ -0,0 +1,3 @@ +```release-note:improvement +container: Upgrade container image to use to Alpine 3.17. +``` diff --git a/Dockerfile b/Dockerfile index 4882f1b46d95..45bef496c237 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # Official docker image that includes binaries from releases.hashicorp.com. This # downloads the release from releases.hashicorp.com and therefore requires that # the release is published before building the Docker image. -FROM docker.mirror.hashicorp.services/alpine:3.15 as official +FROM docker.mirror.hashicorp.services/alpine:3.17 as official # This is the release of Consul to pull in. ARG VERSION @@ -109,7 +109,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"] # Production docker image that uses CI built binaries. # Remember, this image cannot be built locally. -FROM docker.mirror.hashicorp.services/alpine:3.15 as default +FROM docker.mirror.hashicorp.services/alpine:3.17 as default ARG PRODUCT_VERSION ARG BIN_NAME diff --git a/test/integration/connect/envoy/Dockerfile-tcpdump b/test/integration/connect/envoy/Dockerfile-tcpdump index 03116e8f5a27..658cd30a2330 100644 --- a/test/integration/connect/envoy/Dockerfile-tcpdump +++ b/test/integration/connect/envoy/Dockerfile-tcpdump @@ -1,7 +1,7 @@ -FROM alpine:3.12 +FROM alpine:3.17 RUN apk add --no-cache tcpdump VOLUME [ "/data" ] CMD [ "-w", "/data/all.pcap" ] -ENTRYPOINT [ "/usr/sbin/tcpdump" ] +ENTRYPOINT [ "/usr/bin/tcpdump" ] diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 89610337a40d..c7746d9260db 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -584,7 +584,7 @@ function docker_consul_for_proxy_bootstrap { function docker_wget { local DC=$1 shift 1 - docker run --rm --network container:envoy_consul-${DC}_1 docker.mirror.hashicorp.services/alpine:3.9 wget "$@" + docker run --rm --network container:envoy_consul-${DC}_1 docker.mirror.hashicorp.services/alpine:3.17 wget "$@" } function docker_curl { From deaef1d6b4050708a04a8baeada261be324ee676 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 13:09:58 -0500 Subject: [PATCH 026/421] backport of commit 3253386a68ee5a54ab978f0783d5d581a5880dac (#16370) Co-authored-by: Anita Akaeze --- .../consul-container/libs/assert/envoy.go | 18 +- .../consul-container/libs/cluster/cluster.go | 14 ++ .../consul-container/libs/service/helpers.go | 89 +++++--- .../resolver_default_subset_test.go} | 10 +- .../resolver_subset_onlypassing_test.go | 196 ++++++++++++++++++ .../test/upgrade/peering_http_test.go | 3 +- 6 files changed, 299 insertions(+), 31 deletions(-) rename test/integration/consul-container/test/upgrade/{traffic_management_default_subset_test.go => l7_traffic_management/resolver_default_subset_test.go} (98%) create mode 100644 test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go diff --git a/test/integration/consul-container/libs/assert/envoy.go b/test/integration/consul-container/libs/assert/envoy.go index e62118c4f1d8..73677329a5a9 100644 --- a/test/integration/consul-container/libs/assert/envoy.go +++ b/test/integration/consul-container/libs/assert/envoy.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hashicorp/consul/sdk/testutil/retry" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "github.com/hashicorp/go-cleanhttp" "github.com/stretchr/testify/assert" @@ -70,7 +71,11 @@ func AssertUpstreamEndpointStatus(t *testing.T, adminPort int, clusterName, heal filter := fmt.Sprintf(`.cluster_statuses[] | select(.name|contains("%s")) | [.host_statuses[].health_status.eds_health_status] | [select(.[] == "%s")] | length`, clusterName, healthStatus) results, err := utils.JQFilter(clusters, filter) require.NoErrorf(r, err, "could not found cluster name %s", clusterName) - require.Equal(r, count, len(results)) + + resultToString := strings.Join(results, " ") + result, err := strconv.Atoi(resultToString) + assert.NoError(r, err) + require.Equal(r, count, result) }) } @@ -251,3 +256,14 @@ func sanitizeResult(s string) []string { result := strings.Split(strings.ReplaceAll(s, `,`, " "), " ") return append(result[:0], result[1:]...) } + +// AssertServiceHasHealthyInstances asserts the number of instances of service equals count for a given service. +// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-resolver#onlypassing +func AssertServiceHasHealthyInstances(t *testing.T, node libcluster.Agent, service string, onlypassing bool, count int) { + services, _, err := node.GetClient().Health().Service(service, "", onlypassing, nil) + require.NoError(t, err) + for _, v := range services { + fmt.Printf("%s service status: %s\n", v.Service.ID, v.Checks.AggregatedStatus()) + } + require.Equal(t, count, len(services)) +} diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index 2ec57a671733..c569a21a38eb 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -600,6 +600,20 @@ func (c *Cluster) ConfigEntryWrite(entry api.ConfigEntry) error { return err } +func (c *Cluster) ConfigEntryDelete(entry api.ConfigEntry) error { + client, err := c.GetClient(nil, true) + if err != nil { + return err + } + + entries := client.ConfigEntries() + _, err = entries.Delete(entry.GetKind(), entry.GetName(), nil) + if err != nil { + return fmt.Errorf("error deleting config entry: %v", err) + } + return err +} + func extractSecretIDFrom(tokenOutput string) (string, error) { lines := strings.Split(tokenOutput, "\n") for _, line := range lines { diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index 8de49e45c20f..f7b963ff5a0a 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "github.com/hashicorp/consul/api" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" @@ -14,46 +15,38 @@ const ( StaticClientServiceName = "static-client" ) +type Checks struct { + Name string + TTL string +} + +type SidecarService struct { + Port int +} + type ServiceOpts struct { Name string ID string Meta map[string]string HTTPPort int GRPCPort int + Checks Checks + Connect SidecarService } -func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { +// createAndRegisterStaticServerAndSidecar register the services and launch static-server containers +func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int, svc *api.AgentServiceRegistration) (Service, Service, error) { // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. var deferClean utils.ResettableDefer defer deferClean.Execute() - // Register the static-server service and sidecar first to prevent race with sidecar - // trying to get xDS before it's ready - req := &api.AgentServiceRegistration{ - Name: serviceOpts.Name, - ID: serviceOpts.ID, - Port: serviceOpts.HTTPPort, - Connect: &api.AgentServiceConnect{ - SidecarService: &api.AgentServiceRegistration{ - Proxy: &api.AgentServiceConnectProxyConfig{}, - }, - }, - Check: &api.AgentServiceCheck{ - Name: "Static Server Listening", - TCP: fmt.Sprintf("127.0.0.1:%d", serviceOpts.HTTPPort), - Interval: "10s", - Status: api.HealthPassing, - }, - Meta: serviceOpts.Meta, - } - - if err := node.GetClient().Agent().ServiceRegister(req); err != nil { + if err := node.GetClient().Agent().ServiceRegister(svc); err != nil { return nil, nil, err } // Create a service and proxy instance - serverService, err := NewExampleService(context.Background(), serviceOpts.ID, serviceOpts.HTTPPort, serviceOpts.GRPCPort, node) + serverService, err := NewExampleService(context.Background(), svc.ID, svc.Port, grpcPort, node) if err != nil { return nil, nil, err } @@ -61,7 +54,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts _ = serverService.Terminate() }) - serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", serviceOpts.ID), serviceOpts.ID, []int{serviceOpts.HTTPPort}, node) // bindPort not used + serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", svc.ID), svc.ID, []int{svc.Port}, node) // bindPort not used if err != nil { return nil, nil, err } @@ -75,6 +68,54 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts return serverService, serverConnectProxy, nil } +func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { + // Register the static-server service and sidecar first to prevent race with sidecar + // trying to get xDS before it's ready + req := &api.AgentServiceRegistration{ + Name: serviceOpts.Name, + ID: serviceOpts.ID, + Port: serviceOpts.HTTPPort, + Connect: &api.AgentServiceConnect{ + SidecarService: &api.AgentServiceRegistration{ + Proxy: &api.AgentServiceConnectProxyConfig{}, + }, + }, + Check: &api.AgentServiceCheck{ + Name: "Static Server Listening", + TCP: fmt.Sprintf("127.0.0.1:%d", serviceOpts.HTTPPort), + Interval: "10s", + Status: api.HealthPassing, + }, + Meta: serviceOpts.Meta, + } + return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req) +} + +func CreateAndRegisterStaticServerAndSidecarWithChecks(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { + // Register the static-server service and sidecar first to prevent race with sidecar + // trying to get xDS before it's ready + req := &api.AgentServiceRegistration{ + Name: serviceOpts.Name, + ID: serviceOpts.ID, + Port: serviceOpts.HTTPPort, + Connect: &api.AgentServiceConnect{ + SidecarService: &api.AgentServiceRegistration{ + Proxy: &api.AgentServiceConnectProxyConfig{}, + Port: serviceOpts.Connect.Port, + }, + }, + Checks: api.AgentServiceChecks{ + { + Name: serviceOpts.Checks.Name, + TTL: serviceOpts.Checks.TTL, + }, + }, + Meta: serviceOpts.Meta, + } + + return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req) +} + func CreateAndRegisterStaticClientSidecar( node libcluster.Agent, peerName string, diff --git a/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go similarity index 98% rename from test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go rename to test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go index 5ff574e77e91..059ea39ae888 100644 --- a/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go @@ -18,7 +18,7 @@ import ( "gotest.tools/assert" ) -// TestTrafficManagement_Upgrade Summary +// TestTrafficManagement_ServiceResolverDefaultSubset Summary // This test starts up 3 servers and 1 client in the same datacenter. // // Steps: @@ -26,7 +26,7 @@ import ( // - Create one static-server and 2 subsets and 1 client and sidecar, then register them with Consul // - Validate static-server and 2 subsets are and proxy admin endpoint is healthy - 3 instances // - Validate static servers proxy listeners should be up and have right certs -func TestTrafficManagement_ServiceWithSubsets(t *testing.T) { +func TestTrafficManagement_ServiceResolverDefaultSubset(t *testing.T) { t.Parallel() var responseFormat = map[string]string{"format": "json"} @@ -151,8 +151,8 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic GRPCPort: 8079, } _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) - libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "static-server") serviceOptsV1 := &libservice.ServiceOpts{ Name: libservice.StaticServerServiceName, @@ -162,8 +162,8 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic GRPCPort: 8078, } _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) - libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "static-server") serviceOptsV2 := &libservice.ServiceOpts{ Name: libservice.StaticServerServiceName, @@ -173,8 +173,8 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic GRPCPort: 8077, } _, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) - libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "static-server") // Create a client proxy instance with the server as an upstream clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go new file mode 100644 index 000000000000..f33d74d6c1da --- /dev/null +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go @@ -0,0 +1,196 @@ +package upgrade + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + libutils "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestTrafficManagement_ServiceResolverSubsetOnlyPassing Summary +// This test starts up 2 servers and 1 client in the same datacenter. +// +// Steps: +// - Create a single agent cluster. +// - Create one static-server, 1 subset server 1 client and sidecars for all services, then register them with Consul +func TestTrafficManagement_ServiceResolverSubsetOnlyPassing(t *testing.T) { + t.Parallel() + + responseFormat := map[string]string{"format": "json"} + + type testcase struct { + oldversion string + targetVersion string + } + tcs := []testcase{ + { + oldversion: "1.13", + targetVersion: utils.TargetVersion, + }, + { + oldversion: "1.14", + targetVersion: utils.TargetVersion, + }, + } + + run := func(t *testing.T, tc testcase) { + buildOpts := &libcluster.BuildOptions{ + ConsulVersion: tc.oldversion, + Datacenter: "dc1", + InjectAutoEncryption: true, + } + // If version < 1.14 disable AutoEncryption + oldVersion, _ := version.NewVersion(tc.oldversion) + if oldVersion.LessThan(libutils.Version_1_14) { + buildOpts.InjectAutoEncryption = false + } + cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) + node := cluster.Agents[0] + + // Register service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + DefaultSubset: "test", + Subsets: map[string]api.ServiceResolverSubset{ + "test": { + OnlyPassing: true, + }, + }, + ConnectTimeout: 120 * time.Second, + } + err := cluster.ConfigEntryWrite(serviceResolver) + require.NoError(t, err) + + serverConnectProxy, serverConnectProxyV1, clientConnectProxy := createServiceAndSubset(t, cluster) + + _, port := clientConnectProxy.GetAddr() + _, adminPort := clientConnectProxy.GetAdminAddr() + _, serverAdminPort := serverConnectProxy.GetAdminAddr() + _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() + + // Upgrade cluster, restart sidecars then begin service traffic validation + require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) + require.NoError(t, clientConnectProxy.Restart()) + require.NoError(t, serverConnectProxy.Restart()) + require.NoError(t, serverConnectProxyV1.Restart()) + + // force static-server-v1 into a warning state + err = node.GetClient().Agent().UpdateTTL("service:static-server-v1", "", "warn") + assert.NoError(t, err) + + // validate static-client is up and running + libassert.AssertContainerState(t, clientConnectProxy, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + + // validate static-client proxy admin is up + _, clientStatusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, clientStatusCode, fmt.Sprintf("service cannot be reached %v", clientStatusCode)) + + // validate static-server proxy admin is up + _, serverStatusCode, err := libassert.GetEnvoyOutput(serverAdminPort, "stats", responseFormat) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, serverStatusCode, fmt.Sprintf("service cannot be reached %v", serverStatusCode)) + + // validate static-server-v1 proxy admin is up + _, serverStatusCodeV1, err := libassert.GetEnvoyOutput(serverAdminPortV1, "stats", responseFormat) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, serverStatusCodeV1, fmt.Sprintf("service cannot be reached %v", serverStatusCodeV1)) + + // certs are valid + libassert.AssertEnvoyPresentsCertURI(t, adminPort, libservice.StaticClientServiceName) + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPort, libservice.StaticServerServiceName) + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, libservice.StaticServerServiceName) + + // ########################### + // ## with onlypassing=true + // assert only one static-server proxy is healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1) + + // static-client upstream should have 1 healthy endpoint for test.static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "HEALTHY", 1) + + // static-client upstream should have 1 unhealthy endpoint for test.static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "UNHEALTHY", 1) + + // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName) + + // ########################### + // ## with onlypassing=false + // revert to OnlyPassing=false by deleting the config + err = cluster.ConfigEntryDelete(serviceResolver) + require.NoError(t, err) + + // Consul health check assert only one static-server proxy is healthy when onlyPassing is false + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, false, 2) + + // Although the service status is in warning state, when onlypassing is set to false Envoy + // health check returns all service instances with "warning" or "passing" state as Healthy enpoints + libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "HEALTHY", 2) + + // static-client upstream should have 0 unhealthy endpoint for static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "UNHEALTHY", 0) + } + + for _, tc := range tcs { + t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), + func(t *testing.T) { + run(t, tc) + }) + } +} + +// create 2 servers and 1 client +func createServiceAndSubset(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service, libservice.Service) { + node := cluster.Agents[0] + client := node.GetClient() + + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: libservice.StaticServerServiceName, + HTTPPort: 8080, + GRPCPort: 8079, + } + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8081, + GRPCPort: 8078, + Checks: libservice.Checks{ + Name: "main", + TTL: "30m", + }, + Connect: libservice.SidecarService{ + Port: 21011, + }, + } + _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecarWithChecks(node, serviceOptsV1) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + + // Create a client proxy instance with the server as an upstream + clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + + return serverConnectProxy, serverConnectProxyV1, clientConnectProxy +} diff --git a/test/integration/consul-container/test/upgrade/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering_http_test.go index 61ac485d39d5..7c856ff7e4ed 100644 --- a/test/integration/consul-container/test/upgrade/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering_http_test.go @@ -64,10 +64,11 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { GRPCPort: 8078, } _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(c.Clients()[0], serviceOpts) - libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName) if err != nil { return nil, nil, nil, err } + libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName) + err = c.ConfigEntryWrite(&api.ProxyConfigEntry{ Kind: api.ProxyDefaults, Name: "global", From 3cf61fcba88cf8743ecfa7d1e7718da324f0a3b1 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 14:24:39 -0500 Subject: [PATCH 027/421] Backport of Update existing docs from Consul API Gateway -> API Gateway for Kubernetes into release/1.15.x (#16367) * backport of commit acac8d3732c9401b4b4ef380e2c77f6aa5f8f692 * backport of commit 2042464e4cf657392627f46a1b2a805968775fb8 * backport of commit 631ba62d59c3b811be2d94f3476e18302e5e3041 --------- Co-authored-by: Nathan Coleman --- website/content/docs/api-gateway/index.mdx | 4 ++-- website/content/docs/api-gateway/install.mdx | 4 ++-- website/content/docs/api-gateway/tech-specs.mdx | 4 ++-- website/content/docs/api-gateway/upgrades.mdx | 4 ++-- website/content/docs/api-gateway/usage/usage.mdx | 4 ++-- website/data/docs-nav-data.json | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/website/content/docs/api-gateway/index.mdx b/website/content/docs/api-gateway/index.mdx index 1bdc0bfce9d8..462c286e5084 100644 --- a/website/content/docs/api-gateway/index.mdx +++ b/website/content/docs/api-gateway/index.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Consul API Gateway Overview +page_title: API Gateway for Kubernetes Overview description: >- Consul API Gateway enables external network client access to a service mesh on Kubernetes and forwards requests based on path or header information. Learn about how the k8s Gateway API specification configures Consul API Gateway so you can control access and simplify traffic management. --- -# Consul API Gateway Overview +# API Gateway for Kubernetes Overview This topic provides an overview of the Consul API Gateway. diff --git a/website/content/docs/api-gateway/install.mdx b/website/content/docs/api-gateway/install.mdx index 1ee4c61c8fcc..1d715c0c4786 100644 --- a/website/content/docs/api-gateway/install.mdx +++ b/website/content/docs/api-gateway/install.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Install Consul API Gateway +page_title: Install API Gateway for Kubernetes description: >- Learn how to install custom resource definitions (CRDs) and configure the Helm chart so that you can run Consul API Gateway on your Kubernetes deployment. --- -# Install Consul API Gateway +# Install API Gateway for Kubernetes This topic describes how to install and configure Consul API Gateway. diff --git a/website/content/docs/api-gateway/tech-specs.mdx b/website/content/docs/api-gateway/tech-specs.mdx index 9695eb0126a2..7fb142340dec 100644 --- a/website/content/docs/api-gateway/tech-specs.mdx +++ b/website/content/docs/api-gateway/tech-specs.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Consul API Gateway Technical Specifications +page_title: API Gateway for Kubernetes Technical Specifications description: >- Consul API Gateway is a service mesh add-on for Kubernetes deployments. Learn about its requirements for system resources, ports, and component versions, its Enterprise limitations, and compatible k8s cloud environments. --- -# Consul API Gateway Technical Specifications +# API Gateway for Kubernetes Technical Specifications This topic describes the technical specifications associated with using Consul API Gateway. diff --git a/website/content/docs/api-gateway/upgrades.mdx b/website/content/docs/api-gateway/upgrades.mdx index 821a80dce8c5..e67c3a0de9c1 100644 --- a/website/content/docs/api-gateway/upgrades.mdx +++ b/website/content/docs/api-gateway/upgrades.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Upgrade Consul API Gateway +page_title: Upgrade API Gateway for Kubernetes description: >- Upgrade Consul API Gateway to use newly supported features. Learn about the requirements, procedures, and post-configuration changes involved in standard and specific version upgrades. --- -# Upgrade Consul API Gateway +# Upgrade API Gateway for Kubernetes This topic describes how to upgrade Consul API Gateway. diff --git a/website/content/docs/api-gateway/usage/usage.mdx b/website/content/docs/api-gateway/usage/usage.mdx index 6dc4dd576072..b9b864ce234a 100644 --- a/website/content/docs/api-gateway/usage/usage.mdx +++ b/website/content/docs/api-gateway/usage/usage.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Use Consul API Gateway +page_title: Deploy API Gateway for Kubernetes description: >- Learn how to apply a configured Consul API Gateway to your Kubernetes cluster, review the required fields for rerouting HTTP requests, and troubleshoot an error message. --- -# Basic Consul API Gateway Usage +# Deploy API Gateway for Kubernetes This topic describes how to use Consul API Gateway. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 16bf17d84baf..7d39c9434001 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -193,7 +193,7 @@ ] }, { - "title": "Consul API Gateway", + "title": "API Gateway for Kubernetes", "routes": [ { "title": "v0.5.x", @@ -1293,7 +1293,7 @@ "divider": true }, { - "title": "Consul API Gateway", + "title": "API Gateway for Kubernetes", "routes": [ { "title": "Overview", @@ -1315,7 +1315,7 @@ "title": "Usage", "routes": [ { - "title": "Basic Usage", + "title": "Deploy", "path": "api-gateway/usage/usage" }, { From 92bf0a915b8ed57729f0657fa941fcc65fdfe8e8 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 14:54:10 -0500 Subject: [PATCH 028/421] Backport of Add changelog entry for API Gateway (Beta) into release/1.15.x (#16375) * backport of commit ba0ddc3c9eb8dadef3020b64f67a008a8e8e9826 * backport of commit d6d90cb9f422485c9f6020a224337b9b65e04e12 * backport of commit 559e192473e787ea205b1b58c7ee928a9db87e6e --------- Co-authored-by: Nathan Coleman --- .changelog/16369.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/16369.txt diff --git a/.changelog/16369.txt b/.changelog/16369.txt new file mode 100644 index 000000000000..1ae86968c409 --- /dev/null +++ b/.changelog/16369.txt @@ -0,0 +1,3 @@ +```release-note:feature +**API Gateway (Beta)** This version adds support for API gateway on VMs. API gateway provides a highly-configurable ingress for requests coming into a Consul network. For more information, refer to the [API gateway](https://developer.hashicorp.com/consul/docs/connect/gateways/api-gateway) documentation. +``` From bda302b2fadf6b370b19f6d13622f1df3f448c25 Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 22 Feb 2023 14:59:53 -0500 Subject: [PATCH 029/421] Revert "feat: envoy extension - http local rate limit (#16196)" (#16373) This reverts commit e91bc9c0586c5cd11b933f854165189720969340. --- .../builtin/http/localratelimit/copied.go | 58 ---- .../builtin/http/localratelimit/ratelimit.go | 198 -------------- .../http/localratelimit/ratelimit_test.go | 160 ----------- .../envoyextensions/registered_extensions.go | 6 +- agent/xds/delta_envoy_extender_oss_test.go | 21 -- ...cal-ratelimit-applyto-filter.latest.golden | 127 --------- ...cal-ratelimit-applyto-filter.latest.golden | 75 ----- ...cal-ratelimit-applyto-filter.latest.golden | 256 ------------------ ...cal-ratelimit-applyto-filter.latest.golden | 5 - api/config_entry.go | 5 +- go.mod | 2 +- .../envoy/case-envoyext-ratelimit/capture.sh | 4 - .../case-envoyext-ratelimit/service_s1.hcl | 16 -- .../case-envoyext-ratelimit/service_s2.hcl | 5 - .../envoy/case-envoyext-ratelimit/setup.sh | 46 ---- .../envoy/case-envoyext-ratelimit/vars.sh | 3 - .../envoy/case-envoyext-ratelimit/verify.bats | 57 ---- 17 files changed, 5 insertions(+), 1039 deletions(-) delete mode 100644 agent/envoyextensions/builtin/http/localratelimit/copied.go delete mode 100644 agent/envoyextensions/builtin/http/localratelimit/ratelimit.go delete mode 100644 agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go delete mode 100644 agent/xds/testdata/builtin_extension/clusters/http-local-ratelimit-applyto-filter.latest.golden delete mode 100644 agent/xds/testdata/builtin_extension/endpoints/http-local-ratelimit-applyto-filter.latest.golden delete mode 100644 agent/xds/testdata/builtin_extension/listeners/http-local-ratelimit-applyto-filter.latest.golden delete mode 100644 agent/xds/testdata/builtin_extension/routes/http-local-ratelimit-applyto-filter.latest.golden delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/capture.sh delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/service_s1.hcl delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/service_s2.hcl delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/setup.sh delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/vars.sh delete mode 100644 test/integration/connect/envoy/case-envoyext-ratelimit/verify.bats diff --git a/agent/envoyextensions/builtin/http/localratelimit/copied.go b/agent/envoyextensions/builtin/http/localratelimit/copied.go deleted file mode 100644 index ec1d4988eb75..000000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/copied.go +++ /dev/null @@ -1,58 +0,0 @@ -package localratelimit - -import ( - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" -) - -// This is copied from xds and not put into the shared package because I'm not -// convinced it should be shared. - -func makeUpstreamTLSTransportSocket(tlsContext *envoy_tls_v3.UpstreamTlsContext) (*envoy_core_v3.TransportSocket, error) { - if tlsContext == nil { - return nil, nil - } - return makeTransportSocket("tls", tlsContext) -} - -func makeTransportSocket(name string, config proto.Message) (*envoy_core_v3.TransportSocket, error) { - any, err := anypb.New(config) - if err != nil { - return nil, err - } - return &envoy_core_v3.TransportSocket{ - Name: name, - ConfigType: &envoy_core_v3.TransportSocket_TypedConfig{ - TypedConfig: any, - }, - }, nil -} - -func makeEnvoyHTTPFilter(name string, cfg proto.Message) (*envoy_http_v3.HttpFilter, error) { - any, err := anypb.New(cfg) - if err != nil { - return nil, err - } - - return &envoy_http_v3.HttpFilter{ - Name: name, - ConfigType: &envoy_http_v3.HttpFilter_TypedConfig{TypedConfig: any}, - }, nil -} - -func makeFilter(name string, cfg proto.Message) (*envoy_listener_v3.Filter, error) { - any, err := anypb.New(cfg) - if err != nil { - return nil, err - } - - return &envoy_listener_v3.Filter{ - Name: name, - ConfigType: &envoy_listener_v3.Filter_TypedConfig{TypedConfig: any}, - }, nil -} diff --git a/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go b/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go deleted file mode 100644 index 3ffea6f1ff8f..000000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go +++ /dev/null @@ -1,198 +0,0 @@ -package localratelimit - -import ( - "errors" - "fmt" - "time" - - envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - envoy_ratelimit "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/local_ratelimit/v3" - envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" - envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" - "github.com/golang/protobuf/ptypes/wrappers" - "github.com/hashicorp/go-multierror" - "github.com/mitchellh/mapstructure" - "google.golang.org/protobuf/types/known/durationpb" - - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/envoyextensions/extensioncommon" -) - -type ratelimit struct { - ProxyType string - - // Token bucket of the rate limit - MaxTokens *int - TokensPerFill *int - FillInterval *int - - // Percent of requests to be rate limited - FilterEnabled *uint32 - FilterEnforced *uint32 -} - -var _ extensioncommon.BasicExtension = (*ratelimit)(nil) - -// Constructor follows a specific function signature required for the extension registration. -func Constructor(ext api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) { - var r ratelimit - if name := ext.Name; name != api.BuiltinLocalRatelimitExtension { - return nil, fmt.Errorf("expected extension name 'ratelimit' but got %q", name) - } - - if err := r.fromArguments(ext.Arguments); err != nil { - return nil, err - } - - return &extensioncommon.BasicEnvoyExtender{ - Extension: &r, - }, nil -} - -func (r *ratelimit) fromArguments(args map[string]interface{}) error { - if err := mapstructure.Decode(args, r); err != nil { - return fmt.Errorf("error decoding extension arguments: %v", err) - } - return r.validate() -} - -func (r *ratelimit) validate() error { - var resultErr error - - // NOTE: Envoy requires FillInterval value must be greater than 0. - // If unset, it is considered as 0. - if r.FillInterval == nil { - resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) is missing")) - } else if *r.FillInterval <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) must be greater than 0, got %d", *r.FillInterval)) - } - - // NOTE: Envoy requires MaxToken value must be greater than 0. - // If unset, it is considered as 0. - if r.MaxTokens == nil { - resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens is missing")) - } else if *r.MaxTokens <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens must be greater than 0, got %d", r.MaxTokens)) - } - - // TokensPerFill is allowed to unset. In this case, envoy - // uses its default value, which is 1. - if r.TokensPerFill != nil && *r.TokensPerFill <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("TokensPerFill must be greater than 0, got %d", *r.TokensPerFill)) - } - - if err := validateProxyType(r.ProxyType); err != nil { - resultErr = multierror.Append(resultErr, err) - } - - return resultErr -} - -// CanApply determines if the extension can apply to the given extension configuration. -func (p *ratelimit) CanApply(config *extensioncommon.RuntimeConfig) bool { - // rate limit is only applied to the service itself since the limit is - // aggregated from all downstream connections. - return string(config.Kind) == p.ProxyType && !config.IsUpstream() -} - -// PatchRoute does nothing. -func (p ratelimit) PatchRoute(_ *extensioncommon.RuntimeConfig, route *envoy_route_v3.RouteConfiguration) (*envoy_route_v3.RouteConfiguration, bool, error) { - return route, false, nil -} - -// PatchCluster does nothing. -func (p ratelimit) PatchCluster(_ *extensioncommon.RuntimeConfig, c *envoy_cluster_v3.Cluster) (*envoy_cluster_v3.Cluster, bool, error) { - return c, false, nil -} - -// PatchFilter inserts a http local rate_limit filter at the head of -// envoy.filters.network.http_connection_manager filters -func (p ratelimit) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter) (*envoy_listener_v3.Filter, bool, error) { - if filter.Name != "envoy.filters.network.http_connection_manager" { - return filter, false, nil - } - if typedConfig := filter.GetTypedConfig(); typedConfig == nil { - return filter, false, errors.New("error getting typed config for http filter") - } - - config := envoy_resource_v3.GetHTTPConnectionManager(filter) - if config == nil { - return filter, false, errors.New("error unmarshalling filter") - } - - tokenBucket := envoy_type_v3.TokenBucket{} - - if p.TokensPerFill != nil { - tokenBucket.TokensPerFill = &wrappers.UInt32Value{ - Value: uint32(*p.TokensPerFill), - } - } - if p.MaxTokens != nil { - tokenBucket.MaxTokens = uint32(*p.MaxTokens) - } - - if p.FillInterval != nil { - tokenBucket.FillInterval = durationpb.New(time.Duration(*p.FillInterval) * time.Second) - } - - var FilterEnabledDefault *envoy_core_v3.RuntimeFractionalPercent - if p.FilterEnabled != nil { - FilterEnabledDefault = &envoy_core_v3.RuntimeFractionalPercent{ - DefaultValue: &envoy_type_v3.FractionalPercent{ - Numerator: *p.FilterEnabled, - Denominator: envoy_type_v3.FractionalPercent_HUNDRED, - }, - } - } - - var FilterEnforcedDefault *envoy_core_v3.RuntimeFractionalPercent - if p.FilterEnforced != nil { - FilterEnforcedDefault = &envoy_core_v3.RuntimeFractionalPercent{ - DefaultValue: &envoy_type_v3.FractionalPercent{ - Numerator: *p.FilterEnforced, - Denominator: envoy_type_v3.FractionalPercent_HUNDRED, - }, - } - } - - ratelimitHttpFilter, err := makeEnvoyHTTPFilter( - "envoy.filters.http.local_ratelimit", - &envoy_ratelimit.LocalRateLimit{ - TokenBucket: &tokenBucket, - StatPrefix: "local_ratelimit", - FilterEnabled: FilterEnabledDefault, - FilterEnforced: FilterEnforcedDefault, - }, - ) - - if err != nil { - return filter, false, err - } - - changedFilters := make([]*envoy_http_v3.HttpFilter, 0, len(config.HttpFilters)+1) - - // The ratelimitHttpFilter is inserted as the first element of the http - // filter chain. - changedFilters = append(changedFilters, ratelimitHttpFilter) - changedFilters = append(changedFilters, config.HttpFilters...) - config.HttpFilters = changedFilters - - newFilter, err := makeFilter("envoy.filters.network.http_connection_manager", config) - if err != nil { - return filter, false, errors.New("error making new filter") - } - - return newFilter, true, nil -} - -func validateProxyType(t string) error { - if t != "connect-proxy" { - return fmt.Errorf("unexpected ProxyType %q", t) - } - - return nil -} diff --git a/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go b/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go deleted file mode 100644 index 5c68b1f51ea1..000000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package localratelimit - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/envoyextensions/extensioncommon" -) - -func TestConstructor(t *testing.T) { - makeArguments := func(overrides map[string]interface{}) map[string]interface{} { - m := map[string]interface{}{ - "ProxyType": "connect-proxy", - } - - for k, v := range overrides { - m[k] = v - } - - return m - } - - cases := map[string]struct { - extensionName string - arguments map[string]interface{} - expected ratelimit - ok bool - expectedErrMsg string - }{ - "with no arguments": { - arguments: nil, - ok: false, - }, - "with an invalid name": { - arguments: makeArguments(map[string]interface{}{}), - extensionName: "bad", - ok: false, - }, - "MaxToken is missing": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - }), - expectedErrMsg: "MaxTokens is missing", - ok: false, - }, - "MaxTokens <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 0, - }), - expectedErrMsg: "MaxTokens must be greater than 0", - ok: false, - }, - "FillInterval is missing": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "TokensPerFill": 5, - "MaxTokens": 10, - }), - expectedErrMsg: "FillInterval(in second) is missing", - ok: false, - }, - "FillInterval <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 0, - "TokensPerFill": 5, - "MaxTokens": 10, - }), - expectedErrMsg: "FillInterval(in second) must be greater than 0", - ok: false, - }, - "TokensPerFill <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 0, - "MaxTokens": 10, - }), - expectedErrMsg: "TokensPerFill must be greater than 0", - ok: false, - }, - "FilterEnabled < 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 10, - "FilterEnabled": -1, - }), - expectedErrMsg: "cannot parse 'FilterEnabled', -1 overflows uint", - ok: false, - }, - "FilterEnforced < 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 10, - "FilterEnforced": -1, - }), - expectedErrMsg: "cannot parse 'FilterEnforced', -1 overflows uint", - ok: false, - }, - "valid everything": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "MaxTokens": 20, - "TokensPerFill": 5, - }), - expected: ratelimit{ - ProxyType: "connect-proxy", - MaxTokens: intPointer(20), - FillInterval: intPointer(30), - TokensPerFill: intPointer(5), - }, - ok: true, - }, - } - - for n, tc := range cases { - t.Run(n, func(t *testing.T) { - - extensionName := api.BuiltinLocalRatelimitExtension - if tc.extensionName != "" { - extensionName = tc.extensionName - } - - svc := api.CompoundServiceName{Name: "svc"} - ext := extensioncommon.RuntimeConfig{ - ServiceName: svc, - EnvoyExtension: api.EnvoyExtension{ - Name: extensionName, - Arguments: tc.arguments, - }, - } - - e, err := Constructor(ext.EnvoyExtension) - - if tc.ok { - require.NoError(t, err) - require.Equal(t, &extensioncommon.BasicEnvoyExtender{Extension: &tc.expected}, e) - } else { - require.Error(t, err) - require.Contains(t, err.Error(), tc.expectedErrMsg) - } - }) - } -} - -func intPointer(i int) *int { - return &i -} diff --git a/agent/envoyextensions/registered_extensions.go b/agent/envoyextensions/registered_extensions.go index fed8d3c59e80..c765df7c8381 100644 --- a/agent/envoyextensions/registered_extensions.go +++ b/agent/envoyextensions/registered_extensions.go @@ -4,7 +4,6 @@ import ( "fmt" awslambda "github.com/hashicorp/consul/agent/envoyextensions/builtin/aws-lambda" - "github.com/hashicorp/consul/agent/envoyextensions/builtin/http/localratelimit" "github.com/hashicorp/consul/agent/envoyextensions/builtin/lua" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/extensioncommon" @@ -14,9 +13,8 @@ import ( type extensionConstructor func(api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) var extensionConstructors = map[string]extensionConstructor{ - api.BuiltinLuaExtension: lua.Constructor, - api.BuiltinAWSLambdaExtension: awslambda.Constructor, - api.BuiltinLocalRatelimitExtension: localratelimit.Constructor, + api.BuiltinLuaExtension: lua.Constructor, + api.BuiltinAWSLambdaExtension: awslambda.Constructor, } // ConstructExtension attempts to lookup and build an extension from the registry with the diff --git a/agent/xds/delta_envoy_extender_oss_test.go b/agent/xds/delta_envoy_extender_oss_test.go index f6791ba65435..97a610fd4c53 100644 --- a/agent/xds/delta_envoy_extender_oss_test.go +++ b/agent/xds/delta_envoy_extender_oss_test.go @@ -208,27 +208,6 @@ end`, return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nsFunc, nil, makeLambdaServiceDefaults(true)) }, }, - { - name: "http-local-ratelimit-applyto-filter", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { - ns.Proxy.Config["protocol"] = "http" - ns.Proxy.EnvoyExtensions = []structs.EnvoyExtension{ - { - Name: api.BuiltinLocalRatelimitExtension, - Arguments: map[string]interface{}{ - "ProxyType": "connect-proxy", - "MaxTokens": 3, - "TokensPerFill": 2, - "FillInterval": 10, - "FilterEnabled": 100, - "FilterEnforced": 100, - }, - }, - } - }, nil) - }, - }, } latestEnvoyVersion := xdscommon.EnvoyVersions[0] diff --git a/agent/xds/testdata/builtin_extension/clusters/http-local-ratelimit-applyto-filter.latest.golden b/agent/xds/testdata/builtin_extension/clusters/http-local-ratelimit-applyto-filter.latest.golden deleted file mode 100644 index 6f67c341ddf6..000000000000 --- a/agent/xds/testdata/builtin_extension/clusters/http-local-ratelimit-applyto-filter.latest.golden +++ /dev/null @@ -1,127 +0,0 @@ -{ - "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": {}, - "resourceApiVersion": "V3" - } - }, - "connectTimeout": "5s", - "circuitBreakers": {}, - "outlierDetection": {}, - "commonLbConfig": { - "healthyPanicThreshold": {} - }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ - { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" - }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" - } - } - ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" - }, - "matchSubjectAltNames": [ - { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" - } - ] - } - }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" - } - } - }, - { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": {}, - "resourceApiVersion": "V3" - } - }, - "connectTimeout": "5s", - "circuitBreakers": {}, - "outlierDetection": {}, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ - { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" - }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" - } - } - ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" - }, - "matchSubjectAltNames": [ - { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" - }, - { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" - } - ] - } - }, - "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" - } - } - }, - { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "local_app", - "type": "STATIC", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "local_app", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 8080 - } - } - } - } - ] - } - ] - } - } - ], - "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "nonce": "00000001" -} \ No newline at end of file diff --git a/agent/xds/testdata/builtin_extension/endpoints/http-local-ratelimit-applyto-filter.latest.golden b/agent/xds/testdata/builtin_extension/endpoints/http-local-ratelimit-applyto-filter.latest.golden deleted file mode 100644 index e8e6b94a1a90..000000000000 --- a/agent/xds/testdata/builtin_extension/endpoints/http-local-ratelimit-applyto-filter.latest.golden +++ /dev/null @@ -1,75 +0,0 @@ -{ - "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.2", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - }, - { - "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ - { - "lbEndpoints": [ - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - }, - { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.20.1.2", - "portValue": 8080 - } - } - }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 - } - ] - } - ] - } - ], - "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "nonce": "00000001" -} \ No newline at end of file diff --git a/agent/xds/testdata/builtin_extension/listeners/http-local-ratelimit-applyto-filter.latest.golden b/agent/xds/testdata/builtin_extension/listeners/http-local-ratelimit-applyto-filter.latest.golden deleted file mode 100644 index 29c336296e40..000000000000 --- a/agent/xds/testdata/builtin_extension/listeners/http-local-ratelimit-applyto-filter.latest.golden +++ /dev/null @@ -1,256 +0,0 @@ -{ - "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "db:127.0.0.1:9191", - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 9191 - } - }, - "filterChains": [ - { - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.db.default.default.dc1", - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" - } - } - ] - } - ], - "trafficDirection": "OUTBOUND" - }, - { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "prepared_query:geo-cache:127.10.10.10:8181", - "address": { - "socketAddress": { - "address": "127.10.10.10", - "portValue": 8181 - } - }, - "filterChains": [ - { - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.prepared_query_geo-cache", - "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" - } - } - ] - } - ], - "trafficDirection": "OUTBOUND" - }, - { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "public_listener:0.0.0.0:9999", - "address": { - "socketAddress": { - "address": "0.0.0.0", - "portValue": 9999 - } - }, - "filterChains": [ - { - "filters": [ - { - "name": "envoy.filters.network.http_connection_manager", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "statPrefix": "public_listener", - "routeConfig": { - "name": "public_listener", - "virtualHosts": [ - { - "name": "public_listener", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app" - } - } - ] - } - ] - }, - "httpFilters": [ - { - "name": "envoy.filters.http.local_ratelimit", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit", - "statPrefix": "local_ratelimit", - "tokenBucket": { - "maxTokens": 3, - "tokensPerFill": 2, - "fillInterval": "10s" - }, - "filterEnabled": { - "defaultValue": { - "numerator": 100 - } - }, - "filterEnforced": { - "defaultValue": { - "numerator": 100 - } - } - } - }, - { - "name": "envoy.filters.http.rbac", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", - "rules": {} - } - }, - { - "name": "envoy.filters.http.header_to_metadata", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.header_to_metadata.v3.Config", - "requestRules": [ - { - "header": "x-forwarded-client-cert", - "onHeaderPresent": { - "metadataNamespace": "consul", - "key": "trust-domain", - "regexValueRewrite": { - "pattern": { - "googleRe2": {}, - "regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*" - }, - "substitution": "\\1" - } - } - }, - { - "header": "x-forwarded-client-cert", - "onHeaderPresent": { - "metadataNamespace": "consul", - "key": "partition", - "regexValueRewrite": { - "pattern": { - "googleRe2": {}, - "regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*" - }, - "substitution": "\\2" - } - } - }, - { - "header": "x-forwarded-client-cert", - "onHeaderPresent": { - "metadataNamespace": "consul", - "key": "namespace", - "regexValueRewrite": { - "pattern": { - "googleRe2": {}, - "regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*" - }, - "substitution": "\\3" - } - } - }, - { - "header": "x-forwarded-client-cert", - "onHeaderPresent": { - "metadataNamespace": "consul", - "key": "datacenter", - "regexValueRewrite": { - "pattern": { - "googleRe2": {}, - "regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*" - }, - "substitution": "\\4" - } - } - }, - { - "header": "x-forwarded-client-cert", - "onHeaderPresent": { - "metadataNamespace": "consul", - "key": "service", - "regexValueRewrite": { - "pattern": { - "googleRe2": {}, - "regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*" - }, - "substitution": "\\5" - } - } - } - ] - } - }, - { - "name": "envoy.filters.http.router", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" - } - } - ], - "tracing": { - "randomSampling": {} - }, - "forwardClientCertDetails": "APPEND_FORWARD", - "setCurrentClientCertDetails": { - "subject": true, - "cert": true, - "chain": true, - "dns": true, - "uri": true - } - } - } - ], - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ - { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" - }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" - } - } - ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" - } - }, - "alpnProtocols": [ - "http/1.1" - ] - }, - "requireClientCertificate": true - } - } - } - ], - "trafficDirection": "INBOUND" - } - ], - "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", - "nonce": "00000001" -} \ No newline at end of file diff --git a/agent/xds/testdata/builtin_extension/routes/http-local-ratelimit-applyto-filter.latest.golden b/agent/xds/testdata/builtin_extension/routes/http-local-ratelimit-applyto-filter.latest.golden deleted file mode 100644 index d08109381030..000000000000 --- a/agent/xds/testdata/builtin_extension/routes/http-local-ratelimit-applyto-filter.latest.golden +++ /dev/null @@ -1,5 +0,0 @@ -{ - "versionInfo": "00000001", - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" -} \ No newline at end of file diff --git a/api/config_entry.go b/api/config_entry.go index 39b7727c89a4..4e9682ee6f81 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -33,9 +33,8 @@ const ( ) const ( - BuiltinAWSLambdaExtension string = "builtin/aws/lambda" - BuiltinLuaExtension string = "builtin/lua" - BuiltinLocalRatelimitExtension string = "builtin/http/localratelimit" + BuiltinAWSLambdaExtension string = "builtin/aws/lambda" + BuiltinLuaExtension string = "builtin/lua" ) type ConfigEntry interface { diff --git a/go.mod b/go.mod index 299e682647c0..d591c4dd09a8 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/fsnotify/fsnotify v1.5.1 github.com/go-openapi/runtime v0.24.1 github.com/go-openapi/strfmt v0.21.3 - github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.8 github.com/google/gofuzz v1.2.0 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 @@ -158,6 +157,7 @@ require ( github.com/go-ozzo/ozzo-validation v3.6.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/capture.sh b/test/integration/connect/envoy/case-envoyext-ratelimit/capture.sh deleted file mode 100644 index 1a11f7d5e014..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/capture.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -snapshot_envoy_admin localhost:19000 s1 primary || true -snapshot_envoy_admin localhost:19001 s2 primary || true diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/service_s1.hcl b/test/integration/connect/envoy/case-envoyext-ratelimit/service_s1.hcl deleted file mode 100644 index 0d8957c000b5..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/service_s1.hcl +++ /dev/null @@ -1,16 +0,0 @@ -services { - name = "s1" - port = 8080 - connect { - sidecar_service { - proxy { - upstreams = [ - { - destination_name = "s2" - local_bind_port = 5000 - } - ] - } - } - } -} diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/service_s2.hcl b/test/integration/connect/envoy/case-envoyext-ratelimit/service_s2.hcl deleted file mode 100644 index 9c23e79c7267..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/service_s2.hcl +++ /dev/null @@ -1,5 +0,0 @@ -services { - name = "s2" - port = 8181 - connect { sidecar_service {} } -} diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/setup.sh b/test/integration/connect/envoy/case-envoyext-ratelimit/setup.sh deleted file mode 100644 index c49c39d6010b..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/setup.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -eEuo pipefail - -upsert_config_entry primary ' -Kind = "service-defaults" -Name = "s2" -Protocol = "http" -EnvoyExtensions = [ - { - Name = "builtin/http/localratelimit", - Arguments = { - ProxyType = "connect-proxy" - MaxTokens = 1, - TokensPerFill = 1, - FillInterval = 120, - FilterEnabled = 100, - FilterEnforced = 100, - } - } -] -' - -upsert_config_entry primary ' -Kind = "service-defaults" -Name = "s1" -Protocol = "tcp" -EnvoyExtensions = [ - { - Name = "builtin/http/localratelimit", - Arguments = { - ProxyType = "connect-proxy" - MaxTokens = 1, - TokensPerFill = 1, - FillInterval = 120, - FilterEnabled = 100, - FilterEnforced = 100, - } - } -] -' - -register_services primary - -gen_envoy_bootstrap s1 19000 primary -gen_envoy_bootstrap s2 19001 primary diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/vars.sh b/test/integration/connect/envoy/case-envoyext-ratelimit/vars.sh deleted file mode 100644 index 433e50c1b43d..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/vars.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -export REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy" diff --git a/test/integration/connect/envoy/case-envoyext-ratelimit/verify.bats b/test/integration/connect/envoy/case-envoyext-ratelimit/verify.bats deleted file mode 100644 index b8303348e135..000000000000 --- a/test/integration/connect/envoy/case-envoyext-ratelimit/verify.bats +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "s1 proxy admin is up on :19000" { - retry_default curl -f -s localhost:19000/stats -o /dev/null -} - -@test "s2 proxy admin is up on :19001" { - retry_default curl -f -s localhost:19001/stats -o /dev/null -} - -@test "s1 proxy listener should be up and have right cert" { - assert_proxy_presents_cert_uri localhost:21000 s1 -} - -@test "s2 proxy listener should be up and have right cert" { - assert_proxy_presents_cert_uri localhost:21001 s2 -} - -@test "s2 proxy should be healthy" { - assert_service_has_healthy_instances s2 1 -} - -@test "s1 upstream should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1 -} - -@test "s2 proxy should have been configured with http local ratelimit filters" { - HTTP_FILTERS=$(get_envoy_http_filters localhost:19001) - PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') - - echo "HTTP_FILTERS = $HTTP_FILTERS" - echo "PUB = $PUB" - - [ "$PUB" = "envoy.filters.http.local_ratelimit,envoy.filters.http.rbac,envoy.filters.http.header_to_metadata,envoy.filters.http.router" ] -} - -@test "s1(tcp) proxy should not be changed by http/localratelimit extension" { - TCP_FILTERS=$(get_envoy_listener_filters localhost:19000) - PUB=$(echo "$TCP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') - - echo "TCP_FILTERS = $TCP_FILTERS" - echo "PUB = $PUB" - - [ "$PUB" = "envoy.filters.network.rbac,envoy.filters.network.tcp_proxy" ] -} - -@test "first connection to s2 - success" { - run retry_default curl -s -f -d hello localhost:5000 - [ "$status" -eq 0 ] - [[ "$output" == *"hello"* ]] -} - -@test "ratelimit to s2 is in effect - return code 429" { - retry_default must_fail_http_connection localhost:5000 429 -} From 61aeb81919db78ac71dc990164d09bf0eb5cc491 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 15:24:20 -0500 Subject: [PATCH 030/421] backport of commit 181bee38c0500ada717dbf43621a40520ef64e50 (#16378) Co-authored-by: Andrew Stucki --- agent/consul/gateways/controller_gateways.go | 52 ++-- .../gateways/controller_gateways_test.go | 259 +++++++++++++++++- agent/structs/config_entry_status.go | 10 + 3 files changed, 286 insertions(+), 35 deletions(-) diff --git a/agent/consul/gateways/controller_gateways.go b/agent/consul/gateways/controller_gateways.go index b1197d0ba4a5..cfc5a25ba7e0 100644 --- a/agent/consul/gateways/controller_gateways.go +++ b/agent/consul/gateways/controller_gateways.go @@ -409,7 +409,6 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. } var triggerOnce sync.Once - validTargets := true for _, service := range route.GetServiceNames() { _, chainSet, err := store.ReadDiscoveryChainConfigEntries(ws, service.Name, pointerTo(service.EnterpriseMeta)) if err != nil { @@ -422,11 +421,6 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. r.controller.AddTrigger(req, ws.WatchCtx) }) - if chainSet.IsEmpty() { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(errServiceDoesNotExist)) - continue - } - // make sure that we can actually compile a discovery chain based on this route // the main check is to make sure that all of the protocols align chain, err := discoverychain.Compile(discoverychain.CompileRequest{ @@ -438,28 +432,16 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. Entries: chainSet, }) if err != nil { - // we only really need to return the first error for an invalid - // discovery chain, but we still want to set watches on everything in the - // store - if validTargets { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(err)) - validTargets = false - } + updater.SetCondition(conditions.routeInvalidDiscoveryChain(err)) continue } if chain.Protocol != string(route.GetProtocol()) { - if validTargets { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(errInvalidProtocol)) - validTargets = false - } + updater.SetCondition(conditions.routeInvalidDiscoveryChain(errInvalidProtocol)) continue } - // this makes sure we don't override an already set status - if validTargets { - updater.SetCondition(conditions.routeAccepted()) - } + updater.SetCondition(conditions.routeAccepted()) } // if we have no upstream targets, then set the route as invalid @@ -467,17 +449,6 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. // we'll do it here too just in case if len(route.GetServiceNames()) == 0 { updater.SetCondition(conditions.routeNoUpstreams()) - validTargets = false - } - - if !validTargets { - // we return early, but need to make sure we're removed from all referencing - // gateways and our status is updated properly - updated := []*structs.BoundAPIGatewayConfigEntry{} - for _, modifiedGateway := range removeRoute(requestToResourceRef(req), meta...) { - updated = append(updated, modifiedGateway.BoundGateway) - } - return finalize(updated) } // the route is valid, attempt to bind it to all gateways @@ -575,6 +546,8 @@ type gatewayMeta struct { // the map values are pointers so that we can update them directly // and have the changes propagate back to the container gateways. boundListeners map[string]*structs.BoundAPIGatewayListener + + generator *gatewayConditionGenerator } // getAllGatewayMeta returns a pre-constructed list of all valid gateway and state @@ -701,11 +674,22 @@ func (g *gatewayMeta) bindRoute(listener *structs.APIGatewayListener, bound *str return false, nil } + // check to make sure we're not binding to an invalid gateway + if !g.Gateway.Status.MatchesConditionStatus(g.generator.gatewayAccepted()) { + return false, fmt.Errorf("failed to bind route to gateway %s: gateway has not been accepted", g.Gateway.Name) + } + + // check to make sure we're not binding to an invalid route + status := route.GetStatus() + if !status.MatchesConditionStatus(g.generator.routeAccepted()) { + return false, fmt.Errorf("failed to bind route to gateway %s: route has not been accepted", g.Gateway.Name) + } + if route, ok := route.(*structs.HTTPRouteConfigEntry); ok { // check our hostnames hostnames := route.FilteredHostnames(listener.GetHostname()) if len(hostnames) == 0 { - return false, fmt.Errorf("failed to bind route to gateway %s: listener %s is does not have any hostnames that match the route", route.GetName(), g.Gateway.Name) + return false, fmt.Errorf("failed to bind route to gateway %s: listener %s is does not have any hostnames that match the route", g.Gateway.Name, listener.Name) } } @@ -809,6 +793,8 @@ func (g *gatewayMeta) setConflicts(updater *structs.StatusUpdater) { // initialize sets up the listener maps that we use for quickly indexing the listeners in our binding logic func (g *gatewayMeta) initialize() *gatewayMeta { + g.generator = newGatewayConditionGenerator() + // set up the maps for fast access g.boundListeners = make(map[string]*structs.BoundAPIGatewayListener, len(g.BoundGateway.Listeners)) for i, listener := range g.BoundGateway.Listeners { diff --git a/agent/consul/gateways/controller_gateways_test.go b/agent/consul/gateways/controller_gateways_test.go index 0c47809c763a..805a5738ce66 100644 --- a/agent/consul/gateways/controller_gateways_test.go +++ b/agent/consul/gateways/controller_gateways_test.go @@ -49,6 +49,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, route: &structs.TCPRouteConfigEntry{ @@ -61,6 +66,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, expectedBoundGateway: structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -116,6 +126,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, route: &structs.TCPRouteConfigEntry{ @@ -127,6 +142,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Name: "Gateway", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, expectedBoundGateway: structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -417,6 +437,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -431,6 +456,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -521,6 +551,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, { @@ -541,6 +576,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -560,6 +600,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -624,6 +669,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -638,6 +688,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -691,6 +746,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -705,6 +765,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -770,6 +835,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -784,6 +854,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -843,6 +918,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, { @@ -871,6 +951,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -890,6 +975,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -968,6 +1058,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -982,6 +1077,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -1027,6 +1127,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, { @@ -1047,6 +1152,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -1061,6 +1171,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 1", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Name: "TCP Route 2", @@ -1072,6 +1187,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -1128,6 +1248,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolHTTP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -1142,6 +1267,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{}, @@ -1181,6 +1311,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -1195,6 +1330,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -1289,6 +1429,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, }, }, @@ -1302,6 +1447,11 @@ func TestBindRoutesToGateways(t *testing.T) { Kind: structs.APIGateway, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{}, @@ -1430,7 +1580,7 @@ func TestAPIGatewayController(t *testing.T) { }, }, }, - "tcp-route-no-gateways-invalid-targets": { + "tcp-route-not-accepted-bind": { requests: []controller.Request{{ Kind: structs.TCPRoute, Name: "tcp-route", @@ -1444,6 +1594,27 @@ func TestAPIGatewayController(t *testing.T) { Services: []structs.TCPService{{ Name: "tcp-upstream", }}, + Parents: []structs.ResourceReference{{ + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + }}, + }, + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{{ + Name: "listener", + Port: 80, + }}, + }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{{ + Name: "listener", + }}, }, }, finalEntries: []structs.ConfigEntry{ @@ -1453,10 +1624,41 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeInvalidDiscoveryChain(errServiceDoesNotExist), + conditions.routeAccepted(), + conditions.routeUnbound(structs.ResourceReference{ + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + }, errors.New("failed to bind route to gateway api-gateway: gateway has not been accepted")), + }, + }, + }, + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{{ + Name: "listener", + Port: 80, + }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "api-gateway", + SectionName: "listener", + EnterpriseMeta: *defaultMeta, + }), }, }, }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{{ + Name: "listener", + }}, + }, }, }, "tcp-route-no-gateways-invalid-targets-bad-protocol": { @@ -1748,6 +1950,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -1763,6 +1970,11 @@ func TestAPIGatewayController(t *testing.T) { Protocol: structs.ListenerProtocolTCP, Port: 22, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().gatewayAccepted(), + }, + }, }, &structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -1840,6 +2052,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -1931,6 +2148,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Kind: structs.TCPRoute, @@ -1944,6 +2166,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -2061,6 +2288,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, @@ -2076,6 +2308,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -2174,6 +2411,14 @@ func TestAPIGatewayController(t *testing.T) { Kind: structs.TCPRoute, Name: "tcp-route", Meta: acl.DefaultEnterpriseMeta(), + }, { + Kind: structs.TCPRoute, + Name: "tcp-route", + Meta: acl.DefaultEnterpriseMeta(), + }, { + Kind: structs.HTTPRoute, + Name: "http-route", + Meta: acl.DefaultEnterpriseMeta(), }, { Kind: structs.HTTPRoute, Name: "http-route", @@ -2327,6 +2572,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Kind: structs.TCPRoute, @@ -2340,6 +2590,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + newGatewayConditionGenerator().routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, diff --git a/agent/structs/config_entry_status.go b/agent/structs/config_entry_status.go index 5485a6ccaf95..85140f3e5aae 100644 --- a/agent/structs/config_entry_status.go +++ b/agent/structs/config_entry_status.go @@ -50,6 +50,16 @@ type Status struct { Conditions []Condition } +func (s *Status) MatchesConditionStatus(condition Condition) bool { + for _, c := range s.Conditions { + if c.IsCondition(&condition) && + c.Status == condition.Status { + return true + } + } + return false +} + func (s Status) SameConditions(other Status) bool { if len(s.Conditions) != len(other.Conditions) { return false From 76f2bc5c4ca12ab6d0b99ef4a126c681a5371b2e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 15:27:46 -0500 Subject: [PATCH 031/421] Backport of Documentation update: Adding K8S clusters to external Consul servers into release/1.15.x (#16380) * backport of commit 7281e9c2d2a83853587db458d35c49fe366f5fc5 * backport of commit bb2b8535943112c5cd317da4e25c253da6ddc60f * backport of commit de5485371425bd0da323ff4577a3c69fed4d8107 * backport of commit 505b8e1e28f22fb6716ec3b99271a477f082967a --------- Co-authored-by: Ranjandas --- .../servers-outside-kubernetes.mdx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx b/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx index 93c98d7f3171..8050e4d01b68 100644 --- a/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx +++ b/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Join External Servers to Consul on Kubernetes +page_title: Join Kubernetes Clusters to external Consul Servers description: >- - Client agents that run on Kubernetes pods can join existing clusters whose server agents run outside of k8s. Learn how to expose gossip ports and bootstrap ACLs by configuring the Helm chart. + Kubernetes clusters can be joined to existing Consul clusters in a much simpler way with the introduction of Consul Dataplane. Learn how to add Kubernetes Clusters into an existing Consul cluster and bootstrap ACLs by configuring the Helm chart. --- -# Join External Servers to Consul on Kubernetes +# Join Kubernetes Clusters to external Consul Servers If you have a Consul cluster already running, you can configure your Consul on Kubernetes installation to join this existing cluster. @@ -14,9 +14,7 @@ The below `values.yaml` file shows how to configure the Helm chart to install Consul so that it joins an existing Consul server cluster. The `global.enabled` value first disables all chart components by default -so that each component is opt-in. This allows us to _only_ setup the client -agents. We then opt-in to the client agents by setting `client.enabled` to -`true`. +so that each component is opt-in. Next, configure `externalServers` to point it to Consul servers. The `externalServers.hosts` value must be provided and should be set to a DNS, an IP, @@ -37,8 +35,10 @@ externalServers: - **Note:** To join Consul on Kubernetes to an existing Consul server cluster running outside of Kubernetes, -refer to [Consul servers outside of Kubernetes](/consul/docs/k8s/deployment-configurations/servers-outside-kubernetes). +With the introduction of [Consul Dataplane](/consul/docs/connect/dataplane#what-is-consul-dataplane), Consul installation on Kubernetes is simplified by removing the Consul Client agents. +This requires the Helm installation and rest of the consul-k8s components installed on Kubernetes to talk to Consul Servers directly on various ports. +Before starting the installation, ensure that the Consul Servers are configured to have the gRPC port enabled `8502/tcp` using the [`ports.grpc = 8502`](/consul/docs/agent/config/config-files#grpc) configuration option. + ## Configuring TLS @@ -68,7 +68,7 @@ externalServers: If your HTTPS port is different from Consul's default `8501`, you must also set -`externalServers.httpsPort`. +`externalServers.httpsPort`. If the Consul servers are not running TLS enabled, use this config to set the HTTP port the servers are configured with (default `8500`). ## Configuring ACLs From 8ba11d73804360e300df2cac8b3142b5642f7b75 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 15:58:17 -0500 Subject: [PATCH 032/421] backport of commit 7e236479c060839630cda24d6d1b93b33cf95450 (#16272) Co-authored-by: cskh --- .../consul-container/libs/assert/envoy.go | 2 +- .../consul-container/libs/service/connect.go | 7 ++++ .../consul-container/libs/service/examples.go | 7 ++++ .../consul-container/libs/service/gateway.go | 7 ++++ .../consul-container/libs/service/service.go | 1 + .../libs/topology/peering_topology.go | 42 +++++++++++++++---- .../rotate_server_and_ca_then_fail_test.go | 2 +- .../upgrade/peering_control_plane_mgw_test.go | 26 ++++-------- .../test/upgrade/peering_http_test.go | 2 +- 9 files changed, 67 insertions(+), 29 deletions(-) diff --git a/test/integration/consul-container/libs/assert/envoy.go b/test/integration/consul-container/libs/assert/envoy.go index 73677329a5a9..72c0ba990fbf 100644 --- a/test/integration/consul-container/libs/assert/envoy.go +++ b/test/integration/consul-container/libs/assert/envoy.go @@ -132,7 +132,7 @@ func AssertEnvoyMetricAtLeast(t *testing.T, adminPort int, prefix, metric string err error ) failer := func() *retry.Timer { - return &retry.Timer{Timeout: 30 * time.Second, Wait: 500 * time.Millisecond} + return &retry.Timer{Timeout: 60 * time.Second, Wait: 500 * time.Millisecond} } retry.RunWith(failer(), t, func(r *retry.R) { diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 5a1121b88600..49a340bd2a16 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -112,6 +112,13 @@ func (g ConnectContainer) Start() error { return g.container.Start(g.ctx) } +func (g ConnectContainer) Stop() error { + if g.container == nil { + return fmt.Errorf("container has not been initialized") + } + return g.container.Stop(context.Background(), nil) +} + func (g ConnectContainer) Terminate() error { return cluster.TerminateContainer(g.ctx, g.container, true) } diff --git a/test/integration/consul-container/libs/service/examples.go b/test/integration/consul-container/libs/service/examples.go index 836c7ea5138d..3d6258eaa9b3 100644 --- a/test/integration/consul-container/libs/service/examples.go +++ b/test/integration/consul-container/libs/service/examples.go @@ -105,6 +105,13 @@ func (g exampleContainer) Start() error { return g.container.Start(context.Background()) } +func (g exampleContainer) Stop() error { + if g.container == nil { + return fmt.Errorf("container has not been initialized") + } + return g.container.Stop(context.Background(), nil) +} + func (c exampleContainer) Terminate() error { return cluster.TerminateContainer(c.ctx, c.container, true) } diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 4aabdb918d4c..7028a612928c 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -90,6 +90,13 @@ func (g gatewayContainer) Start() error { return g.container.Start(context.Background()) } +func (g gatewayContainer) Stop() error { + if g.container == nil { + return fmt.Errorf("container has not been initialized") + } + return g.container.Stop(context.Background(), nil) +} + func (c gatewayContainer) Terminate() error { return cluster.TerminateContainer(c.ctx, c.container, true) } diff --git a/test/integration/consul-container/libs/service/service.go b/test/integration/consul-container/libs/service/service.go index 2e706bfccaf2..f32bd67ff8b2 100644 --- a/test/integration/consul-container/libs/service/service.go +++ b/test/integration/consul-container/libs/service/service.go @@ -19,6 +19,7 @@ type Service interface { GetName() string GetServiceName() string Start() (err error) + Stop() (err error) Terminate() error Restart() error GetStatus() (string, error) diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 1c764c45c53c..ba36978c72f4 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -41,6 +41,7 @@ type BuiltCluster struct { func BasicPeeringTwoClustersSetup( t *testing.T, consulVersion string, + peeringThroughMeshgateway bool, ) (*BuiltCluster, *BuiltCluster) { // acceptingCluster, acceptingCtx, acceptingClient := NewPeeringCluster(t, "dc1", 3, consulVersion, true) acceptingCluster, acceptingCtx, acceptingClient := NewPeeringCluster(t, 3, &libcluster.BuildOptions{ @@ -53,6 +54,38 @@ func BasicPeeringTwoClustersSetup( ConsulVersion: consulVersion, InjectAutoEncryption: true, }) + + // Create the mesh gateway for dataplane traffic and peering control plane traffic (if enabled) + acceptingClusterGateway, err := libservice.NewGatewayService(context.Background(), "mesh", "mesh", acceptingCluster.Clients()[0]) + require.NoError(t, err) + dialingClusterGateway, err := libservice.NewGatewayService(context.Background(), "mesh", "mesh", dialingCluster.Clients()[0]) + require.NoError(t, err) + + // Enable peering control plane traffic through mesh gateway + if peeringThroughMeshgateway { + req := &api.MeshConfigEntry{ + Peering: &api.PeeringMeshConfig{ + PeerThroughMeshGateways: true, + }, + } + configCluster := func(cli *api.Client) error { + libassert.CatalogServiceExists(t, cli, "mesh") + ok, _, err := cli.ConfigEntries().Set(req, &api.WriteOptions{}) + if !ok { + return fmt.Errorf("config entry is not set") + } + + if err != nil { + return fmt.Errorf("error writing config entry: %s", err) + } + return nil + } + err = configCluster(dialingClient) + require.NoError(t, err) + err = configCluster(acceptingClient) + require.NoError(t, err) + } + require.NoError(t, dialingCluster.PeerWithCluster(acceptingClient, AcceptingPeerName, DialingPeerName)) libassert.PeeringStatus(t, acceptingClient, AcceptingPeerName, api.PeeringStateActive) @@ -60,7 +93,6 @@ func BasicPeeringTwoClustersSetup( // Register an static-server service in acceptingCluster and export to dialing cluster var serverService, serverSidecarService libservice.Service - var acceptingClusterGateway libservice.Service { clientNode := acceptingCluster.Clients()[0] @@ -81,15 +113,10 @@ func BasicPeeringTwoClustersSetup( libassert.CatalogServiceExists(t, acceptingClient, "static-server-sidecar-proxy") require.NoError(t, serverService.Export("default", AcceptingPeerName, acceptingClient)) - - // Create the mesh gateway for dataplane traffic - acceptingClusterGateway, err = libservice.NewGatewayService(context.Background(), "mesh", "mesh", clientNode) - require.NoError(t, err) } // Register an static-client service in dialing cluster and set upstream to static-server service var clientSidecarService *libservice.ConnectContainer - var dialingClusterGateway libservice.Service { clientNode := dialingCluster.Clients()[0] @@ -100,9 +127,6 @@ func BasicPeeringTwoClustersSetup( libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy") - // Create the mesh gateway for dataplane traffic - dialingClusterGateway, err = libservice.NewGatewayService(context.Background(), "mesh", "mesh", clientNode) - require.NoError(t, err) } _, adminPort := clientSidecarService.GetAdminAddr() diff --git a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go index 223effa449b2..bbac9cc03401 100644 --- a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go +++ b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go @@ -50,7 +50,7 @@ import ( func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.TargetVersion) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.TargetVersion, false) var ( acceptingCluster = accepting.Cluster dialingCluster = dialing.Cluster diff --git a/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go b/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go index f4112b6f6b83..5ccba9567739 100644 --- a/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go +++ b/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go @@ -42,7 +42,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { } run := func(t *testing.T, tc testcase) { - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, tc.oldversion) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, tc.oldversion, true) var ( acceptingCluster = accepting.Cluster dialingCluster = dialing.Cluster @@ -54,19 +54,6 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { acceptingClient, err := acceptingCluster.GetClient(nil, false) require.NoError(t, err) - // Enable peering control plane traffic through mesh gateway - req := &api.MeshConfigEntry{ - Peering: &api.PeeringMeshConfig{ - PeerThroughMeshGateways: true, - }, - } - ok, _, err := dialingClient.ConfigEntries().Set(req, &api.WriteOptions{}) - require.True(t, ok) - require.NoError(t, err) - ok, _, err = acceptingClient.ConfigEntries().Set(req, &api.WriteOptions{}) - require.True(t, ok) - require.NoError(t, err) - // Verify control plane endpoints and traffic in gateway _, gatewayAdminPort := dialing.Gateway.GetAdminAddr() libassert.AssertUpstreamEndpointStatus(t, gatewayAdminPort, "server.dc1.peering", "HEALTHY", 1) @@ -74,6 +61,9 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { libassert.AssertEnvoyMetricAtLeast(t, gatewayAdminPort, "cluster.static-server.default.default.accepting-to-dialer.external", "upstream_cx_total", 1) + libassert.AssertEnvoyMetricAtLeast(t, gatewayAdminPort, + "cluster.server.dc1.peering", + "upstream_cx_total", 1) // Upgrade the accepting cluster and assert peering is still ACTIVE require.NoError(t, acceptingCluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) @@ -90,11 +80,12 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { // - Register a new static-client service in dialing cluster and // - set upstream to static-server service in peered cluster - // Restart the gateway & proxy sidecar + // Stop the accepting gateway and restart dialing gateway + // to force peering control plane traffic through dialing mesh gateway + require.NoError(t, accepting.Gateway.Stop()) require.NoError(t, dialing.Gateway.Restart()) - require.NoError(t, dialing.Container.Restart()) - // Restarted gateway should not have any measurement on data plane traffic + // Restarted dialing gateway should not have any measurement on data plane traffic libassert.AssertEnvoyMetricAtMost(t, gatewayAdminPort, "cluster.static-server.default.default.accepting-to-dialer.external", "upstream_cx_total", 0) @@ -102,6 +93,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { libassert.AssertEnvoyMetricAtLeast(t, gatewayAdminPort, "cluster.server.dc1.peering", "upstream_cx_total", 1) + require.NoError(t, accepting.Gateway.Start()) clientSidecarService, err := libservice.CreateAndRegisterStaticClientSidecar(dialingCluster.Servers()[0], libtopology.DialingPeerName, true) require.NoError(t, err) diff --git a/test/integration/consul-container/test/upgrade/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering_http_test.go index 7c856ff7e4ed..e799cf59a8ac 100644 --- a/test/integration/consul-container/test/upgrade/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering_http_test.go @@ -221,7 +221,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { } run := func(t *testing.T, tc testcase) { - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, tc.oldversion) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, tc.oldversion, false) var ( acceptingCluster = accepting.Cluster dialingCluster = dialing.Cluster From 7cd0eff8571721f35c434fb08fe4cd7539e0b6b3 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 16:25:30 -0500 Subject: [PATCH 033/421] fix failed cherry pick (#16303) Co-authored-by: Maliz --- agent/xds/validateupstream-test/validateupstream_test.go | 2 +- command/troubleshoot/upstreams/troubleshoot_upstreams.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/xds/validateupstream-test/validateupstream_test.go b/agent/xds/validateupstream-test/validateupstream_test.go index 250b6acdec3b..6b758f853831 100644 --- a/agent/xds/validateupstream-test/validateupstream_test.go +++ b/agent/xds/validateupstream-test/validateupstream_test.go @@ -9,7 +9,7 @@ import ( "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" - "github.com/hashicorp/consul/troubleshoot/proxy" + troubleshoot "github.com/hashicorp/consul/troubleshoot/proxy" testinf "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" ) diff --git a/command/troubleshoot/upstreams/troubleshoot_upstreams.go b/command/troubleshoot/upstreams/troubleshoot_upstreams.go index 435630cdc9e9..8f941699ce0c 100644 --- a/command/troubleshoot/upstreams/troubleshoot_upstreams.go +++ b/command/troubleshoot/upstreams/troubleshoot_upstreams.go @@ -77,7 +77,7 @@ func (c *cmd) Run(args []string) int { return 1 } - c.UI.HeaderOutput(fmt.Sprintf("Upstreams (explicit upstreams only) (%v)\n", len(envoyIDs))) + c.UI.HeaderOutput(fmt.Sprintf("Upstreams (explicit upstreams only) (%v)", len(envoyIDs))) for _, u := range envoyIDs { c.UI.UnchangedOutput(u) } From 926d480ff021239695a80e47516415e7a299f144 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 16:52:48 -0500 Subject: [PATCH 034/421] Backport of Docs/rate limiting 1.15 into release/1.15.x (#16384) * backport of commit 5042d8d753510777a323c287098ef5c9ab2cfed8 * backport of commit c6b83c4fff6539329cbaae9e4b993396191e4482 * backport of commit b12a56941e5d80e64fa220455795f59a308b89d3 * backport of commit 16d81ddb6d034bf310c976c59b090f93d20fea2c * backport of commit 48ff8f7bca375dd689207ad4a29a27da244c6e7b * backport of commit e677bc76faa2d1fbecbd5a36c63447d72bdd6812 * backport of commit 74924a228c8ba6ebda8dfb641e0d761c30f36a79 * backport of commit fe9bca727f60c6e2e1e45af004739a9b3343ad5d --------- Co-authored-by: trujillo-adam --- .../docs/agent/config/config-files.mdx | 16 +-- website/content/docs/agent/config/index.mdx | 2 +- website/content/docs/agent/index.mdx | 27 +++-- website/content/docs/agent/limits/index.mdx | 30 +++++ .../docs/agent/limits/init-rate-limits.mdx | 32 ++++++ .../limits/set-global-traffic-rate-limits.mdx | 107 ++++++++++++++++++ website/content/docs/agent/telemetry.mdx | 4 +- website/data/docs-nav-data.json | 17 +++ 8 files changed, 212 insertions(+), 23 deletions(-) create mode 100644 website/content/docs/agent/limits/index.mdx create mode 100644 website/content/docs/agent/limits/init-rate-limits.mdx create mode 100644 website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 82053a935674..92047ce897e7 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -534,17 +534,17 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `license_path` This specifies the path to a file that contains the Consul Enterprise license. Alternatively the license may also be specified in either the `CONSUL_LICENSE` or `CONSUL_LICENSE_PATH` environment variables. See the [licensing documentation](/consul/docs/enterprise/license/overview) for more information about Consul Enterprise license management. Added in versions 1.10.0, 1.9.7 and 1.8.13. Prior to version 1.10.0 the value may be set for all agents to facilitate forwards compatibility with 1.10 but will only actually be used by client agents. -- `limits` Available in Consul 0.9.3 and later, this is a nested - object that configures limits that are enforced by the agent. Prior to Consul 1.5.2, - this only applied to agents in client mode, not Consul servers. The following parameters - are available: +- `limits`: This block specifies various types of limits that the Consul server agent enforces. - `http_max_conns_per_client` - Configures a limit of how many concurrent TCP connections a single client IP address is allowed to open to the agent's HTTP(S) server. This affects the HTTP(S) servers in both client and server agents. Default value is `200`. - `https_handshake_timeout` - Configures the limit for how long the HTTPS server in both client and server agents will wait for a client to complete a TLS handshake. This should be kept conservative as it limits how many connections an unauthenticated attacker can open if `verify_incoming` is being using to authenticate clients (strongly recommended in production). Default value is `5s`. - - `request_limits` - This object povides configuration for rate limiting RPC and gRPC requests on the consul server. As a result of rate limiting gRPC and RPC request, HTTP requests to the Consul server are rate limited. - - `mode` - Configures whether rate limiting is enabled or not as well as how it behaves through the use of 3 possible modes. The default value of "disabled" will prevent any rate limiting from occuring. A value of "permissive" will cause the system to track requests against the `read_rate` and `write_rate` but will only log violations and will not block and will allow the request to continue processing. A value of "enforcing" also tracks requests against the `read_rate` and `write_rate` but in addition to logging violations, the system will block the request from processings by returning an error. - - `read_rate` - Configures how frequently RPC, gRPC, and HTTP queries are allowed to happen. The rate limiter limits the rate to tokens per second equal to this value. See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. - - `write_rate` - Configures how frequently RPC, gRPC, and HTTP write are allowed to happen. The rate limiter limits the rate to tokens per second equal to this value. See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. + - `request_limits` - This object specifies configurations that limit the rate of RPC and gRPC requests on the Consul server. Limiting the rate of gRPC and RPC requests also limits HTTP requests to the Consul server. + - `mode` - String value that specifies an action to take if the rate of requests exceeds the limit. You can specify the following values: + - `permissive`: The server continues to allow requests and records an error in the logs. + - `enforcing`: The server stops accepting requests and records an error in the logs. + - `disabled`: Limits are not enforced or tracked. This is the default value for `mode`. + - `read_rate` - Integer value that specifies the number of read requests per second. Default is `100`. + - `write_rate` - Integer value that specifies the number of write requests per second. Default is `100`. - `rpc_handshake_timeout` - Configures the limit for how long servers will wait after a client TCP connection is established before they complete the connection handshake. When TLS is used, the same timeout applies to the TLS handshake separately from the initial protocol negotiation. All Consul clients should perform this immediately on establishing a new connection. This should be kept conservative as it limits how many connections an unauthenticated attacker can open if `verify_incoming` is being using to authenticate clients (strongly recommended in production). When `verify_incoming` is true on servers, this limits how long the connection socket and associated goroutines will be held open before the client successfully authenticates. Default value is `5s`. - `rpc_client_timeout` - Configures the limit for how long a client is allowed to read from an RPC connection. This is used to set an upper bound for calls to eventually terminate so that RPC connections are not held indefinitely. Blocking queries can override this timeout. Default is `60s`. - `rpc_max_conns_per_client` - Configures a limit of how many concurrent TCP connections a single source IP address is allowed to open to a single server. It affects both clients connections and other server connections. In general Consul clients multiplex many RPC calls over a single TCP connection so this can typically be kept low. It needs to be more than one though since servers open at least one additional connection for raft RPC, possibly more for WAN federation when using network areas, and snapshot requests from clients run over a separate TCP conn. A reasonably low limit significantly reduces the ability of an unauthenticated attacker to consume unbounded resources by holding open many connections. You may need to increase this if WAN federated servers connect via proxies or NAT gateways or similar causing many legitimate connections from a single source IP. Default value is `100` which is designed to be extremely conservative to limit issues with certain deployment patterns. Most deployments can probably reduce this safely. 100 connections on modern server hardware should not cause a significant impact on resource usage from an unauthenticated attacker though. diff --git a/website/content/docs/agent/config/index.mdx b/website/content/docs/agent/config/index.mdx index b2e3ac42c835..0ea4a030fb7f 100644 --- a/website/content/docs/agent/config/index.mdx +++ b/website/content/docs/agent/config/index.mdx @@ -72,7 +72,7 @@ The following agent configuration options are reloadable at runtime: - These can be important in certain outage situations so being able to control them without a restart provides a recovery path that doesn't involve downtime. They generally shouldn't be changed otherwise. -- [RPC rate limiting](/consul/docs/agent/config/config-files#limits) +- [RPC rate limits](/consul/docs/agent/config/config-files#limits) - [HTTP Maximum Connections per Client](/consul/docs/agent/config/config-files#http_max_conns_per_client) - Services - TLS Configuration diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index bab9138e50da..380602081a91 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -33,7 +33,7 @@ The following process describes the agent lifecycle within the context of an exi As a result, all nodes will eventually become aware of each other. 1. **Existing servers will begin replicating to the new node** if the agent is a server. -### Failures and Crashes +### Failures and crashes In the event of a network failure, some nodes may be unable to reach other nodes. Unreachable nodes will be marked as _failed_. @@ -48,7 +48,7 @@ catalog. Once the network recovers or a crashed agent restarts, the cluster will repair itself and unmark a node as failed. The health check in the catalog will also be updated to reflect the current state. -### Exiting Nodes +### Exiting nodes When a node leaves a cluster, it communicates its intent and the cluster marks the node as having _left_. In contrast to changes related to failures, all of the services provided by a node are immediately deregistered. @@ -61,6 +61,9 @@ interval of 72 hours (changing the reap interval is _not_ recommended due to its consequences during outage situations). Reaping is similar to leaving, causing all associated services to be deregistered. +## Limit traffic rates +You can define a set of rate limiting configurations that help operators protect Consul servers from excessive or peak usage. The configurations enable you to gracefully degrade Consul servers to avoid a global interruption of service. You can allocate a set of resources to different Consul users and eliminate the risks that some users consuming too many resources pose to others. Consul supports global server rate limiting, which lets configure Consul servers to deny requests that exceed the read or write limits. Refer to [Traffic Rate Limits Overview](/consul/docs/agent/limits/limit-traffic-rates). + ## Requirements You should run one Consul agent per server or host. @@ -73,7 +76,7 @@ Refer to the following sections for information about host, port, memory, and ot The [Datacenter Deploy tutorial](/consul/tutorials/production-deploy/reference-architecture#deployment-system-requirements) contains additional information, including licensing configuration, environment variables, and other details. -### Maximum Latency Network requirements +### Maximum latency network requirements Consul uses the gossip protocol to share information across agents. To function properly, you cannot exceed the protocol's maximum latency threshold. The latency threshold is calculated according to the total round trip time (RTT) for communication between all agents. Other network usages outside of Gossip are not bound by these latency requirements (i.e. client to server RPCs, HTTP API requests, xDS proxy configuration, DNS). @@ -82,7 +85,7 @@ For data sent between all Consul agents the following latency requirements must - Average RTT for all traffic cannot exceed 50ms. - RTT for 99 percent of traffic cannot exceed 100ms. -## Starting the Consul Agent +## Starting the Consul agent Start a Consul agent with the `consul` command and `agent` subcommand using the following syntax: @@ -111,7 +114,7 @@ $ consul agent -data-dir=tmp/consul -dev Agents are highly configurable, which enables you to deploy Consul to any infrastructure. Many of the default options for the `agent` command are suitable for becoming familiar with a local instance of Consul. In practice, however, several additional configuration options must be specified for Consul to function as expected. Refer to [Agent Configuration](/consul/docs/agent/config) topic for a complete list of configuration options. -### Understanding the Agent Startup Output +### Understanding the agent startup output Consul prints several important messages on startup. The following example shows output from the [`consul agent`](/consul/commands/agent) command: @@ -162,7 +165,7 @@ When running under `systemd` on Linux, Consul notifies systemd by sending this either the `join` or `retry_join` option has to be set and the service definition file has to have `Type=notify` set. -## Configuring Consul Agents +## Configuring Consul agents You can specify many options to configure how Consul operates when issuing the `consul agent` command. You can also create one or more configuration files and provide them to Consul at startup using either the `-config-file` or `-config-dir` option. @@ -180,7 +183,7 @@ $ consul agent -config-file=server.json The configuration options necessary to successfully use Consul depend on several factors, including the type of agent you are configuring (client or server), the type of environment you are deploying to (e.g., on-premise, multi-cloud, etc.), and the security options you want to implement (ACLs, gRPC encryption). The following examples are intended to help you understand some of the combinations you can implement to configure Consul. -### Common Configuration Settings +### Common configuration settings The following settings are commonly used in the configuration file (also called a service definition file when registering services with Consul) to configure Consul agents: @@ -195,7 +198,7 @@ The following settings are commonly used in the configuration file (also called | `addresses` | Block of nested objects that define addresses bound to the agent for internal cluster communication. | `"http": "0.0.0.0"` See the Agent Configuration page for [default address values](/consul/docs/agent/config/config-files#addresses) | | `ports` | Block of nested objects that define ports bound to agent addresses.
See (link to addresses option) for details. | See the Agent Configuration page for [default port values](/consul/docs/agent/config/config-files#ports) | -### Server Node in a Service Mesh +### Server node in a service mesh The following example configuration is for a server agent named "`consul-server`". The server is [bootstrapped](/consul/docs/agent/config/cli-flags#_bootstrap) and the Consul GUI is enabled. The reason this server agent is configured for a service mesh is that the `connect` configuration is enabled. Connect is Consul's service mesh component that provides service-to-service connection authorization and encryption using mutual Transport Layer Security (TLS). Applications can use sidecar proxies in a service mesh configuration to establish TLS connections for inbound and outbound connections without being aware of Connect at all. See [Connect](/consul/docs/connect) for details. @@ -243,7 +246,7 @@ connect { -### Server Node with Encryption Enabled +### Server node with encryption enabled The following example shows a server node configured with encryption enabled. Refer to the [Security](/consul/docs/security) chapter for additional information about how to configure security options for Consul. @@ -313,7 +316,7 @@ tls { -### Client Node Registering a Service +### Client node registering a service Using Consul as a central service registry is a common use case. The following example configuration includes common settings to register a service with a Consul agent and enable health checks (see [Checks](/consul/docs/discovery/checks) to learn more about health checks): @@ -371,7 +374,7 @@ service { -## Client Node with Multiple Interfaces or IP addresses +## Client node with multiple interfaces or IP addresses The following example shows how to configure Consul to listen on multiple interfaces or IP addresses using a [go-sockaddr template]. @@ -422,7 +425,7 @@ advertise_addr = "{{ GetInterfaceIP \"en0\" }}" -## Stopping an Agent +## Stopping an agent An agent can be stopped in two ways: gracefully or forcefully. Servers and Clients both behave differently depending on the leave that is performed. There diff --git a/website/content/docs/agent/limits/index.mdx b/website/content/docs/agent/limits/index.mdx new file mode 100644 index 000000000000..9d88f5f4b0c2 --- /dev/null +++ b/website/content/docs/agent/limits/index.mdx @@ -0,0 +1,30 @@ +--- +layout: docs +page_title: Limit Traffic Rates Overview +description: Rate limiting is a set of Consul server agent configurations that you can use to mitigate the risks to Consul servers when clients send excessive requests to Consul resources. + +--- + +# Limit Traffic Rates Overview +This topic provides overview information about the traffic rates limits you can configure for Consul servers. + +## Introduction +You can configure global RPC rate limits to mitigate the risks to Consul servers when clients send excessive read or write requests to Consul resources. A read request is defined as any request that does not modify Consul internal state. A write request is defined as any request that modifies Consul internal state. Read and write requests are limited separately. + +## Rate limit modes +You can set one of the following modes, which determine how Consul servers react when the request limits are exceeded. + +- **Enforcing mode**: In this mode, the rate limiter denies requests to a server beyond a configurable rate. Consul generates metrics and logs to help operators understand their Consul load and configure limits accordingly. +- **Permissive mode**: The rate limiter allows requests if the limits are reached and produces metrics and logs to help operators understand their Consul load and configure limits accordingly. This mode is intended to help you configure limits and debug specific issues. +- **Disabled mode**: Disables the rate limiter. All requests are allowed and no logs or metrics are produced. This is the default mode. + +Refer to [`rate_limits`](/consul/docs/agent/config/config-files#request_limits) for additional configuration information. + +## Request denials +When an HTTP request is denied for rate limiting reason, Consul returns one of the following errors: + +- **429 Resource Exhausted**: Indicates that a server is not able to perform the request but that another server could potentially fulfill it. This error is most common on stale reads because any server may fulfill state read requests. To resolve this type of error, we recommend immediately retrying the request to another server. If the request came from a Consul client agent, the agent automatically retries the request up to the limit set in the [`rpc_hold_timeout`](/consul/docs/agent/config/config-files#rpc_hold_timeout) configuration . + +- **503 Service Unavailable**: Indicates that server is unable to perform the request and that no other server can fulfill the request, either. This usually occurs on consistent reads or for writes. In this case we recommend retrying according to an exponential backoff schedule. If the request came from a Consul client agent, the agent automatically retries the request according to the [`rpc_hold_timeout`](/consul/docs/agent/config/config-files#rpc_hold_timeout) configuration. + +Refer to [Rate limit reached on the server](/consul/docs/troubleshoot/common-errors#rate-limit-reached-on-the-server) for additional information. \ No newline at end of file diff --git a/website/content/docs/agent/limits/init-rate-limits.mdx b/website/content/docs/agent/limits/init-rate-limits.mdx new file mode 100644 index 000000000000..5f9b5ac0580e --- /dev/null +++ b/website/content/docs/agent/limits/init-rate-limits.mdx @@ -0,0 +1,32 @@ +--- +layout: docs +page_title: Initialize Rate Limit Settings +description: Learn how to determins regular and peak loads in your network so that you can set the initial global rate limit configurations. +--- + +# Initialize Rate Limit Settings + +In order to set limits for traffic, you must first understand regular and peak loads in your network. We recommend completing the following steps to benchmark request rates in your environment so that you can implement limits appropriate for your applications. + +1. Specify a global rate limit with arbitrary values in the agent configuration file based on the following conditions: + + - Environment where Consul servers are running + - Number of servers and the projected load + - Existing metrics expressing requests per second + +1. Set the `mode` to `permissive`. In the following example, Consul agents are allowed up to 1000 reads and 500 writes per second: + + ```hcl + request_limits { + mode = "permissive" + read_rate = 1000.0 + write_rate =500.0 + } + ``` + +1. Observe the logs and metrics for your application's typical cycle, such as a 24 hour period. Refer to [`log_file`](/consul/docs/agent/config/config-files#log_file) for information about where to retrieve logs. Call the [`/agent/metrics`](/consul/api-docs/agent#view-metrics) HTTP API endpoint and check the data for the following metrics: + + - `rpc.rate_limit.exceeded.read` + - `rpc.rate_limit.exceeded.write` + +1. If the limits are not reached, set the `mode` configuration to `enforcing`. Otherwise adjust and iterate limits. \ No newline at end of file diff --git a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx new file mode 100644 index 000000000000..6e14ea2b1cba --- /dev/null +++ b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx @@ -0,0 +1,107 @@ +--- +layout: docs +page_title: Set a Global Limit on Traffic Rates +description: Use global rate limits to prevent excessive rates of requests to Consul servers. +--- +# Set a Global Limit on Traffic Rates + +This topic describes how to configure rate limits for RPC and gRPC traffic to the Consul server. + +## Introduction +Rate limits apply to each Consul server separately and are intended to limit the number of read requests or write requests to the server on the RPC and internal gRPC endpoints. + +Because all requests coming to a Consul server eventually perform an RPC or an internal gRPC request, these limits also apply to other user interfaces, such as the HTTP API interface, the CLI, and the external gRPC endpoint for services in the Consul service mesh. + +Refer to [Initialize Rate Limit Settings]() for additional information about right-sizing your gRPC request configurations. + +## Set a global rate limit for a Consul server +Configure the following settings in your Consul server configuration to limit the RPC and gRPC traffic rates. + +- Set the rate limiter [`mode`](/consul/docs/agent/config/config-files#mode-1) +- Set the [`read_rate`](/consul/docs/agent/config/config-files#read_rate) +- Set the [`write_rate`](/consul/docs/agent/config/config-files#write_rate) + +In the following example, the Consul server is configured to prevent more than `500` read and `200` write RPC calls: + + + +```hcl +limits = { + rate_limit = { + mode = "enforcing" + read_rate = 500 + write_rate = 200 + } +} +``` + +```json +{ + "limits" : { + "rate_limit" : { + "mode" : "enforcing", + "read_rate" : 500, + "write_rate" : 200 + } + } +} + +``` + + + +## Access rate limit logs +Consul prints a log line for each rate limit request. The log provides the information necessary for identifying the source of the request and the configured limit. Consul prints the log `DEBUG` log level and can drop the log to avoid affecting the server health. Dropping a log line increments the `rpc.rate_limit.log_dropped` metric. + +The following example log shows that RPC request from `127.0.0.1:53562` to `KVS.Apply` exceeded the limit: + + + +```shell-session +2023-02-17T10:01:15.565-0500 [DEBUG] agent.server.rpc-rate-limit: RPC +exceeded allowed rate limit: rpc=KVS.Apply source_addr=127.0.0.1:53562 +limit_type=global/write limit_enforced=false +``` + + +## Review rate limit metrics +Consul captures the following metrics associated with rate limits: + +- Type of limit +- Operation +- Rate limit mode + +Call the `agent/metrics` API endpoint to view the metrics associated with rate limits. Refer to [View Metrics](/consul/api-docs/agent#view-metrics) for API usage information. In the following example, Consul dropped a call to the `consul` service because it exceeded the limit by one call: + +```shell-session +$ curl http://127.0.0.1:8500/v1/agent/metrics +{ + . . . + "Counters": [ + { + "Name": "consul.rpc.rate_limit.exceeded", + "Count": 1, + "Sum": 1, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": { + "service": "consul" + } + }, + { + "Name": "consul.rpc.rate_limit.log_dropped", + "Count": 1, + "Sum": 1, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": {} + } + ], + . . . +``` + +Refer to [Telemetry]() for additional information. diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index 53d7ed917663..b234c01179a0 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -477,8 +477,8 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.transition.heartbeat_timeout` | The number of times an agent has transitioned to the Candidate state, after receive no heartbeat messages from the last known leader. | timeouts / interval | counter | | `consul.raft.verify_leader` | This metric doesn't have a direct correlation to the leader change. It just counts the number of times an agent checks if it is still the leader or not. For example, during every consistent read, the check is done. Depending on the load in the system, this metric count can be high as it is incremented each time a consistent read is completed. | checks / interval | Counter | | `consul.rpc.accept_conn` | Increments when a server accepts an RPC connection. | connections | counter | -| `consul.rpc.rate_limit.exceeded` | Increments whenever an RPC is over a configured rate limit. In permissive mode, the RPC is still allowed to proceed. | RPCs | counter | -| `consul.rpc.rate_limit.log_dropped` | Increments whenever a log that is emitted because an RPC exceeded a rate limit gets dropped because the output buffer is full. | log messages dropped | counter | +| `consul.rpc.rate_limit.exceeded` | Number of rate limited requests. Only increments when `rate_limits.mode` is set to `permissive` or `enforcing`. | requests | counter | +| `consul.rpc.rate_limit.log_dropped` | Number of rate limited requests logs dropped for performance reasons. Only increments when `rate_limits.mode` is set to `permissive` or `enforcing` and the log is unable to print the number of excessive requests. | log lines | counter | | `consul.catalog.register` | Measures the time it takes to complete a catalog register operation. | ms | timer | | `consul.catalog.deregister` | Measures the time it takes to complete a catalog deregister operation. | ms | timer | | `consul.server.isLeader` | Track if a server is a leader(1) or not(0) | 1 or 0 | gauge | diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 7d39c9434001..a15b5169ac29 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -734,6 +734,23 @@ } ] }, + { + "title": "Limit Traffic Rates", + "routes": [ + { + "title": "Overview", + "path": "agent/limits" + }, + { + "title": "Initialize Rate Limit Settings", + "path": "agent/limits/init-rate-limits" + }, + { + "title": "Set Global Traffic Rate Limits", + "path": "agent/limits/set-global-traffic-rate-limits" + } + ] + }, { "title": "Configuration Entries", "path": "agent/config-entries" From 2fb0a2f16e94c5baa1e493cbd3ce6d6d3aec0c1d Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 16:56:28 -0500 Subject: [PATCH 035/421] Backport of Add docs for usage endpoint and command into release/1.15.x (#16383) * backport of commit 92acb57e4d7b1b5a2a34b9c45d5b22c6cca8e635 * backport of commit a97ccea4b0679a8b6643a5877fbce7dc19b566bd --------- Co-authored-by: Kyle Havlovitz --- website/content/api-docs/operator/usage.mdx | 161 ++++++++++++++++++++ website/content/commands/operator/index.mdx | 2 + website/content/commands/operator/usage.mdx | 100 ++++++++++++ website/data/api-docs-nav-data.json | 4 + website/data/commands-nav-data.json | 4 + 5 files changed, 271 insertions(+) create mode 100644 website/content/api-docs/operator/usage.mdx create mode 100644 website/content/commands/operator/usage.mdx diff --git a/website/content/api-docs/operator/usage.mdx b/website/content/api-docs/operator/usage.mdx new file mode 100644 index 000000000000..82202b13fb82 --- /dev/null +++ b/website/content/api-docs/operator/usage.mdx @@ -0,0 +1,161 @@ +--- +layout: api +page_title: Usage - Operator - HTTP API +description: |- + The /operator/usage endpoint returns usage information about the number of + services, service instances and Connect-enabled service instances by + datacenter. +--- + +# Usage Operator HTTP API + +The `/operator/usage` endpoint returns usage information about the number of +services, service instances and Connect-enabled service instances by datacenter. + +| Method | Path | Produces | +| ------ | ----------------- | ------------------ | +| `GET` | `/operator/usage` | `application/json` | + +The table below shows this endpoint's support for +[blocking queries](/consul/api-docs/features/blocking), +[consistency modes](/consul/api-docs/features/consistency), +[agent caching](/consul/api-docs/features/caching), and +[required ACLs](/consul/api-docs/api-structure#authentication). + +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | ------------- | --------------- | +| `YES` | `all` | `none` | `operator:read` | + +The corresponding CLI command is [`consul operator usage instances`](/consul/commands/operator/usage). + +### Query Parameters + +- `global` `(bool: false)` - If present, usage information for all + known datacenters will be returned. By default, only the local datacenter's + usage information is returned. + +- `stale` `(bool: false)` - If the cluster does not currently have a leader, an + error will be returned. You can use the `?stale` query parameter to read the + Raft configuration from any of the Consul servers. + +### Sample Request + +```shell-session +$ curl \ + http://127.0.0.1:8500/v1/operator/usage +``` + +### Sample Response + + + +```json +{ + "Usage": { + "dc1": { + "Services": 1, + "ServiceInstances": 1, + "ConnectServiceInstances": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + }, + "BillableServiceInstances": 0 + } + }, + "Index": 13, + "LastContact": 0, + "KnownLeader": true, + "ConsistencyLevel": "leader", + "NotModified": false, + "Backend": 0, + "ResultsFilteredByACLs": false +} +``` + + +```json +{ + "Usage": { + "dc1": { + "Services": 1, + "ServiceInstances": 1, + "ConnectServiceInstances": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + }, + "BillableServiceInstances": 0, + "PartitionNamespaceServices": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceServiceInstances": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceBillableServiceInstances": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceConnectServiceInstances": { + "default": { + "default": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + } + } + } + } + }, + "Index": 13, + "LastContact": 0, + "KnownLeader": true, + "ConsistencyLevel": "leader", + "NotModified": false, + "Backend": 0, + "ResultsFilteredByACLs": false +} +``` + + + +- `Services` is the total number of unique service names registered in the + datacenter. + +- `ServiceInstances` is the total number of service instances registered in + the datacenter. + +- `ConnectServiceInstances` is the total number of Connect service instances + registered in the datacenter. + +- `BillableServiceInstances` is the total number of billable service instances + registered in the datacenter. This is only relevant in Consul Enterprise + and is the total service instance count, not including any Connect service + instances or any Consul server instances. + +- `PartitionNamespaceServices` is the total number + of unique service names registered in the datacenter, by partition and + namespace. + +- `PartitionNamespaceServiceInstances` is the total + number of service instances registered in the datacenter, by partition and + namespace. + +- `PartitionNamespaceBillableServiceInstances` is + the total number of billable service instances registered in the datacenter, + by partition and namespace. + +- `PartitionNamespaceConnectServiceInstances` is + the total number of Connect service instances registered in the datacenter, + by partition and namespace. \ No newline at end of file diff --git a/website/content/commands/operator/index.mdx b/website/content/commands/operator/index.mdx index 14ba32905283..060c42255880 100644 --- a/website/content/commands/operator/index.mdx +++ b/website/content/commands/operator/index.mdx @@ -37,6 +37,7 @@ Subcommands: area Provides tools for working with network areas (Enterprise-only) autopilot Provides tools for modifying Autopilot configuration raft Provides cluster-level tools for Consul operators + usage Provides cluster-level usage information ``` For more information, examples, and usage about a subcommand, click on the name @@ -45,3 +46,4 @@ of the subcommand in the sidebar or one of the links below: - [area](/consul/commands/operator/area) - [autopilot](/consul/commands/operator/autopilot) - [raft](/consul/commands/operator/raft) +- [usage](/consul/commands/operator/usage) diff --git a/website/content/commands/operator/usage.mdx b/website/content/commands/operator/usage.mdx new file mode 100644 index 000000000000..b0d7d03db2f9 --- /dev/null +++ b/website/content/commands/operator/usage.mdx @@ -0,0 +1,100 @@ +--- +layout: commands +page_title: 'Commands: Operator Usage' +description: > + The operator usage command provides cluster-level tools for Consul operators + to view usage information, such as service and service instance counts. +--- + +# Consul Operator Usage + +Command: `consul operator usage` + +The Usage `operator` command provides cluster-level tools for Consul operators +to view usage information, such as service and service instance counts. + +```text +Usage: consul operator usage [options] + + # ... + +Subcommands: + instances Display service instance usage information +``` + +## instances + +Corresponding HTTP API Endpoint: [\[GET\] /v1/operator/usage](/consul/api-docs/operator/usage#operator-usage) + +This command retrieves usage information about the number of services registered in a given +datacenter. By default, the datacenter of the local agent is queried. + +The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of +[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) +are not supported from commands, but may be from the corresponding HTTP endpoint. + +| ACL Required | +| --------------- | +| `operator:read` | + +Usage: `consul operator usage instances` + +The output looks like this: + +```text +$ consul operator usage instances +Billable Service Instances Total: 3 +dc1 Billable Service Instances: 3 + +Billable Services +Services Service instances +2 3 + +Connect Services +Type Service instances +connect-native 0 +connect-proxy 0 +ingress-gateway 0 +mesh-gateway 1 +terminating-gateway 0 +``` + +With the `-all-datacenters` flag: + +```text +$ consul operator usage instances -all-datacenters +Billable Service Instances Total: 4 +dc1 Billable Service Instances: 3 +dc2 Billable Service Instances: 1 + +Billable Services +Datacenter Services Service instances +dc1 2 3 +dc2 1 1 + +Total 3 4 + +Connect Services +Datacenter Type Service instances +dc1 connect-native 0 +dc1 connect-proxy 0 +dc1 ingress-gateway 0 +dc1 mesh-gateway 1 +dc1 terminating-gateway 0 +dc2 connect-native 0 +dc2 connect-proxy 0 +dc2 ingress-gateway 0 +dc2 mesh-gateway 1 +dc2 terminating-gateway 1 + +Total 3 +``` + +#### Command Options + +- `-all-datacenters` - Display service counts from all known datacenters. + Default is `false`. + +- `-billable` - Display only billable service information. Default is `false`. + +- `-connect` - Display only Connect service information. Default is `false`. diff --git a/website/data/api-docs-nav-data.json b/website/data/api-docs-nav-data.json index fb1dd87421be..66d8fa9a9492 100644 --- a/website/data/api-docs-nav-data.json +++ b/website/data/api-docs-nav-data.json @@ -165,6 +165,10 @@ { "title": "Segment", "path": "operator/segment" + }, + { + "title": "Usage", + "path": "operator/usage" } ] }, diff --git a/website/data/commands-nav-data.json b/website/data/commands-nav-data.json index aac55d7952bb..ee491e9dfa7b 100644 --- a/website/data/commands-nav-data.json +++ b/website/data/commands-nav-data.json @@ -425,6 +425,10 @@ { "title": "raft", "path": "operator/raft" + }, + { + "title": "usage", + "path": "operator/usage" } ] }, From a6180659d0f3d3c3fd4a74de4b98af037dda1afd Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Feb 2023 17:15:34 -0500 Subject: [PATCH 036/421] Backport of Add docs for usage endpoint and command into release/1.15.x (#16382) * backport of commit 92acb57e4d7b1b5a2a34b9c45d5b22c6cca8e635 * backport of commit a97ccea4b0679a8b6643a5877fbce7dc19b566bd --------- Co-authored-by: Kyle Havlovitz From 983a1b8ddb5ce02192aa58bb2d451a5e4140f588 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 08:15:21 -0800 Subject: [PATCH 037/421] backport of commit 1180908144b4afd26c3546d2ab43d34e135a029c (#16389) Co-authored-by: Paul Banks --- agent/consul/server_log_verification.go | 4 +- agent/metrics_test.go | 190 ++++++++++++++++++++++++ agent/setup.go | 61 +++++++- 3 files changed, 248 insertions(+), 7 deletions(-) diff --git a/agent/consul/server_log_verification.go b/agent/consul/server_log_verification.go index 0c7e63e3a12c..cb95b9aeeee8 100644 --- a/agent/consul/server_log_verification.go +++ b/agent/consul/server_log_verification.go @@ -62,12 +62,12 @@ func makeLogVerifyReportFn(logger hclog.Logger) verifier.ReportFn { if r.WrittenSum > 0 && r.WrittenSum != r.ExpectedSum { // The failure occurred before the follower wrote to the log so it // must be corrupted in flight from the leader! - l2.Info("verification checksum FAILED: in-flight corruption", + l2.Error("verification checksum FAILED: in-flight corruption", "followerWriteChecksum", fmt.Sprintf("%08x", r.WrittenSum), "readChecksum", fmt.Sprintf("%08x", r.ReadSum), ) } else { - l2.Info("verification checksum FAILED: storage corruption", + l2.Error("verification checksum FAILED: storage corruption", "followerWriteChecksum", fmt.Sprintf("%08x", r.WrittenSum), "readChecksum", fmt.Sprintf("%08x", r.ReadSum), ) diff --git a/agent/metrics_test.go b/agent/metrics_test.go index 1f649dd07a5b..6f75a4d233b3 100644 --- a/agent/metrics_test.go +++ b/agent/metrics_test.go @@ -432,3 +432,193 @@ func TestHTTPHandlers_AgentMetrics_CACertExpiry_Prometheus(t *testing.T) { }) } + +func TestHTTPHandlers_AgentMetrics_WAL_Prometheus(t *testing.T) { + skipIfShortTesting(t) + // This test cannot use t.Parallel() since we modify global state, ie the global metrics instance + + t.Run("client agent emits nothing", func(t *testing.T) { + hcl := ` + server = false + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_4" + } + raft_logstore { + backend = "wal" + } + bootstrap = false + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + require.NotContains(t, respRec.Body.String(), "agent_4_raft_wal") + }) + + t.Run("server with WAL enabled emits WAL metrics", func(t *testing.T) { + hcl := ` + server = true + bootstrap = true + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_5" + } + connect { + enabled = true + } + raft_logstore { + backend = "wal" + } + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + out := respRec.Body.String() + require.Contains(t, out, "agent_5_raft_wal_head_truncations") + require.Contains(t, out, "agent_5_raft_wal_last_segment_age_seconds") + require.Contains(t, out, "agent_5_raft_wal_log_appends") + require.Contains(t, out, "agent_5_raft_wal_log_entries_read") + require.Contains(t, out, "agent_5_raft_wal_log_entries_written") + require.Contains(t, out, "agent_5_raft_wal_log_entry_bytes_read") + require.Contains(t, out, "agent_5_raft_wal_log_entry_bytes_written") + require.Contains(t, out, "agent_5_raft_wal_segment_rotations") + require.Contains(t, out, "agent_5_raft_wal_stable_gets") + require.Contains(t, out, "agent_5_raft_wal_stable_sets") + require.Contains(t, out, "agent_5_raft_wal_tail_truncations") + }) + + t.Run("server without WAL enabled emits no WAL metrics", func(t *testing.T) { + hcl := ` + server = true + bootstrap = true + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_6" + } + connect { + enabled = true + } + raft_logstore { + backend = "boltdb" + } + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + require.NotContains(t, respRec.Body.String(), "agent_6_raft_wal") + }) + +} + +func TestHTTPHandlers_AgentMetrics_LogVerifier_Prometheus(t *testing.T) { + skipIfShortTesting(t) + // This test cannot use t.Parallel() since we modify global state, ie the global metrics instance + + t.Run("client agent emits nothing", func(t *testing.T) { + hcl := ` + server = false + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_4" + } + raft_logstore { + verification { + enabled = true + interval = "1s" + } + } + bootstrap = false + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + require.NotContains(t, respRec.Body.String(), "agent_4_raft_logstore_verifier") + }) + + t.Run("server with verifier enabled emits all metrics", func(t *testing.T) { + hcl := ` + server = true + bootstrap = true + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_5" + } + connect { + enabled = true + } + raft_logstore { + verification { + enabled = true + interval = "1s" + } + } + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + out := respRec.Body.String() + require.Contains(t, out, "agent_5_raft_logstore_verifier_checkpoints_written") + require.Contains(t, out, "agent_5_raft_logstore_verifier_dropped_reports") + require.Contains(t, out, "agent_5_raft_logstore_verifier_ranges_verified") + require.Contains(t, out, "agent_5_raft_logstore_verifier_read_checksum_failures") + require.Contains(t, out, "agent_5_raft_logstore_verifier_write_checksum_failures") + }) + + t.Run("server with verifier disabled emits no extra metrics", func(t *testing.T) { + hcl := ` + server = true + bootstrap = true + telemetry = { + prometheus_retention_time = "5s", + disable_hostname = true + metrics_prefix = "agent_6" + } + connect { + enabled = true + } + raft_logstore { + verification { + enabled = false + } + } + ` + + a := StartTestAgent(t, TestAgent{HCL: hcl}) + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + respRec := httptest.NewRecorder() + recordPromMetrics(t, a, respRec) + + require.NotContains(t, respRec.Body.String(), "agent_6_raft_logstore_verifier") + }) + +} diff --git a/agent/setup.go b/agent/setup.go index 01d7b7593f62..8dc5e5e18c06 100644 --- a/agent/setup.go +++ b/agent/setup.go @@ -9,6 +9,8 @@ import ( "github.com/armon/go-metrics/prometheus" "github.com/hashicorp/go-hclog" + wal "github.com/hashicorp/raft-wal" + "github.com/hashicorp/raft-wal/verifier" "google.golang.org/grpc/grpclog" autoconf "github.com/hashicorp/consul/agent/auto-config" @@ -89,7 +91,7 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer, providedLogger hcl } isServer := result.RuntimeConfig.ServerMode - gauges, counters, summaries := getPrometheusDefs(cfg.Telemetry, isServer) + gauges, counters, summaries := getPrometheusDefs(cfg, isServer) cfg.Telemetry.PrometheusOpts.GaugeDefinitions = gauges cfg.Telemetry.PrometheusOpts.CounterDefinitions = counters cfg.Telemetry.PrometheusOpts.SummaryDefinitions = summaries @@ -226,7 +228,7 @@ func newConnPool(config *config.RuntimeConfig, logger hclog.Logger, tls *tlsutil // getPrometheusDefs reaches into every slice of prometheus defs we've defined in each part of the agent, and appends // all of our slices into one nice slice of definitions per metric type for the Consul agent to pass to go-metrics. -func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.GaugeDefinition, []prometheus.CounterDefinition, []prometheus.SummaryDefinition) { +func getPrometheusDefs(cfg *config.RuntimeConfig, isServer bool) ([]prometheus.GaugeDefinition, []prometheus.CounterDefinition, []prometheus.SummaryDefinition) { // TODO: "raft..." metrics come from the raft lib and we should migrate these to a telemetry // package within. In the mean time, we're going to define a few here because they're key to monitoring Consul. raftGauges := []prometheus.GaugeDefinition{ @@ -272,6 +274,29 @@ func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.Gau ) } + if isServer && cfg.RaftLogStoreConfig.Verification.Enabled { + verifierGauges := make([]prometheus.GaugeDefinition, 0) + for _, d := range verifier.MetricDefinitions.Gauges { + verifierGauges = append(verifierGauges, prometheus.GaugeDefinition{ + Name: []string{"raft", "logstore", "verifier", d.Name}, + Help: d.Desc, + }) + } + gauges = append(gauges, verifierGauges) + } + + if isServer && cfg.RaftLogStoreConfig.Backend == consul.LogStoreBackendWAL { + + walGauges := make([]prometheus.GaugeDefinition, 0) + for _, d := range wal.MetricDefinitions.Gauges { + walGauges = append(walGauges, prometheus.GaugeDefinition{ + Name: []string{"raft", "wal", d.Name}, + Help: d.Desc, + }) + } + gauges = append(gauges, walGauges) + } + // Flatten definitions // NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique? var gaugeDefs []prometheus.GaugeDefinition @@ -280,7 +305,7 @@ func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.Gau // TODO(kit): Prepending the service to each definition should be handled by go-metrics var withService []prometheus.GaugeDefinition for _, gauge := range g { - gauge.Name = append([]string{cfg.MetricsPrefix}, gauge.Name...) + gauge.Name = append([]string{cfg.Telemetry.MetricsPrefix}, gauge.Name...) withService = append(withService, gauge) } gaugeDefs = append(gaugeDefs, withService...) @@ -316,6 +341,32 @@ func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.Gau raftCounters, rate.Counters, } + + // For some unknown reason, we seem to add the raft counters above without + // checking if this is a server like we do above for some of the summaries + // above. We should probably fix that but I want to not change behavior right + // now. If we are a server, add summaries for WAL and verifier metrics. + if isServer && cfg.RaftLogStoreConfig.Verification.Enabled { + verifierCounters := make([]prometheus.CounterDefinition, 0) + for _, d := range verifier.MetricDefinitions.Counters { + verifierCounters = append(verifierCounters, prometheus.CounterDefinition{ + Name: []string{"raft", "logstore", "verifier", d.Name}, + Help: d.Desc, + }) + } + counters = append(counters, verifierCounters) + } + if isServer && cfg.RaftLogStoreConfig.Backend == consul.LogStoreBackendWAL { + walCounters := make([]prometheus.CounterDefinition, 0) + for _, d := range wal.MetricDefinitions.Counters { + walCounters = append(walCounters, prometheus.CounterDefinition{ + Name: []string{"raft", "wal", d.Name}, + Help: d.Desc, + }) + } + counters = append(counters, walCounters) + } + // Flatten definitions // NOTE(kit): Do we actually want to create a set here so we can ensure definition names are unique? var counterDefs []prometheus.CounterDefinition @@ -323,7 +374,7 @@ func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.Gau // TODO(kit): Prepending the service to each definition should be handled by go-metrics var withService []prometheus.CounterDefinition for _, counter := range c { - counter.Name = append([]string{cfg.MetricsPrefix}, counter.Name...) + counter.Name = append([]string{cfg.Telemetry.MetricsPrefix}, counter.Name...) withService = append(withService, counter) } counterDefs = append(counterDefs, withService...) @@ -377,7 +428,7 @@ func getPrometheusDefs(cfg lib.TelemetryConfig, isServer bool) ([]prometheus.Gau // TODO(kit): Prepending the service to each definition should be handled by go-metrics var withService []prometheus.SummaryDefinition for _, summary := range s { - summary.Name = append([]string{cfg.MetricsPrefix}, summary.Name...) + summary.Name = append([]string{cfg.Telemetry.MetricsPrefix}, summary.Name...) withService = append(withService, summary) } summaryDefs = append(summaryDefs, withService...) From 42913d6613c306fdfa664f9872d41445df2810ae Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 10:45:37 -0800 Subject: [PATCH 038/421] Backport of Docs/cluster peering 1.15 updates into release/1.15.x (#16398) * backport of commit e878d2d3e435a724e26789ab6fda84d009961495 * Docs/cluster peering 1.15 updates (#16291) * initial commit * initial commit * Overview updates * Overview page improvements * More Overview improvements * improvements * Small fixes/updates * Updates * Overview updates * Nav data * More nav updates * Fix * updates * Updates + tip test * Directory test * refining * Create restructure w/ k8s * Single usage page * Technical Specification * k8s pages * typo * L7 traffic management * Manage connections * k8s page fix * Create page tab corrections * link to k8s * intentions * corrections * Add-on intention descriptions * adjustments * Missing * Diagram improvements * Final diagram update * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu * diagram name fix * Fixes * Updates to index.mdx * Tech specs page corrections * Tech specs page rename * update link to tech specs * K8s - new pages + tech specs * k8s - manage peering connections * k8s L7 traffic management * Separated establish connection pages * Directory fixes * Usage clean up * k8s docs edits * Updated nav data * CodeBlock Component fix * filename * CodeBlockConfig removal * Redirects * Update k8s filenames * Reshuffle k8s tech specs for clarity, fmt yaml files * Update general cluster peering docs, reorder CLI > API > UI, cross link to kubernetes * Fix config rendering in k8s usage docs, cross link to general usage from k8s docs * fix legacy link * update k8s docs * fix nested list rendering * redirect fix * page error --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu Co-authored-by: Tu Nguyen --------- Co-authored-by: boruszak Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu Co-authored-by: Tu Nguyen --- website/content/api-docs/operator/usage.mdx | 2 + .../configuration.mdx} | 28 +- .../cluster-peering/create-manage-peering.mdx | 537 ---------------- .../docs/connect/cluster-peering/index.mdx | 74 ++- .../docs/connect/cluster-peering/k8s.mdx | 593 ------------------ .../connect/cluster-peering/tech-specs.mdx | 69 ++ .../usage/establish-cluster-peering.mdx | 271 ++++++++ .../usage/manage-connections.mdx | 137 ++++ .../usage/peering-traffic-management.mdx | 168 +++++ .../config-entries/service-intentions.mdx | 51 ++ .../connect/cluster-peering/tech-specs.mdx | 180 ++++++ .../usage/establish-peering.mdx | 453 +++++++++++++ .../cluster-peering/usage/l7-traffic.mdx | 75 +++ .../cluster-peering/usage/manage-peering.mdx | 121 ++++ website/data/docs-nav-data.json | 57 +- .../public/img/cluster-peering-diagram.png | Bin 0 -> 129219 bytes website/redirects.js | 14 +- 17 files changed, 1654 insertions(+), 1176 deletions(-) rename website/content/docs/connect/{gateways/mesh-gateway/service-to-service-traffic-peers.mdx => cluster-peering/configuration.mdx} (53%) delete mode 100644 website/content/docs/connect/cluster-peering/create-manage-peering.mdx delete mode 100644 website/content/docs/connect/cluster-peering/k8s.mdx create mode 100644 website/content/docs/connect/cluster-peering/tech-specs.mdx create mode 100644 website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx create mode 100644 website/content/docs/connect/cluster-peering/usage/manage-connections.mdx create mode 100644 website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx create mode 100644 website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx create mode 100644 website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx create mode 100644 website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx create mode 100644 website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx create mode 100644 website/public/img/cluster-peering-diagram.png diff --git a/website/content/api-docs/operator/usage.mdx b/website/content/api-docs/operator/usage.mdx index 82202b13fb82..73d9adbb7bf6 100644 --- a/website/content/api-docs/operator/usage.mdx +++ b/website/content/api-docs/operator/usage.mdx @@ -48,6 +48,7 @@ $ curl \ ### Sample Response + ```json { @@ -75,6 +76,7 @@ $ curl \ } ``` + ```json { diff --git a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx b/website/content/docs/connect/cluster-peering/configuration.mdx similarity index 53% rename from website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx rename to website/content/docs/connect/cluster-peering/configuration.mdx index ee5a3e38d386..53fab696c0c2 100644 --- a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx +++ b/website/content/docs/connect/cluster-peering/configuration.mdx @@ -1,19 +1,13 @@ --- layout: docs -page_title: Enabling Service-to-service Traffic Across Peered Clusters +page_title: Cluster Peering Configuration description: >- - Mesh gateways are specialized proxies that route data between services that cannot communicate directly. Learn how to enable service-to-service traffic across clusters in different datacenters or admin partitions that have an established peering connection. + --- # Enabling Service-to-service Traffic Across Peered Clusters -Mesh gateways are required for you to route service mesh traffic between peered Consul clusters. Clusters can reside in different clouds or runtime environments where general interconnectivity between all services in all clusters is not feasible. - -At a minimum, a peered cluster exporting a service must have a mesh gateway registered. -For Enterprise, this mesh gateway must also be registered in the same partition as the exported service(s). -To use the `local` mesh gateway mode, there must also be a mesh gateway regsitered in the importing cluster. - -Unlike mesh gateways for WAN-federated datacenters and partitions, mesh gateways between peers terminate mTLS sessions to decrypt data to HTTP services and then re-encrypt traffic to send to services. Data must be decrypted in order to evaluate and apply dynamic routing rules at the destination cluster, which reduces coupling between peers. +The topic provides an overview of the configuration options and process for cluster peering. ## Prerequisites @@ -21,7 +15,7 @@ To configure mesh gateways for cluster peering, make sure your Consul environmen - Consul version 1.14.0 or newer. - A local Consul agent is required to manage mesh gateway configuration. -- Use [Envoy proxies](/consul/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. +- Use [Envoy proxies](/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. ## Configuration @@ -30,31 +24,33 @@ Configure the following settings to register and use the mesh gateway as a servi ### Gateway registration - Specify `mesh-gateway` in the `kind` field to register the gateway with Consul. -- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For Envoy, refer to the [Gateway Options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional configuration information. +- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For Envoy, refer to the [Gateway Options](/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional configuration information. -Alternatively, you can also use the CLI to spin up and register a gateway in Consul. For additional information, refer to the [`consul connect envoy` command](/consul/commands/connect/envoy#mesh-gateways). +Alternatively, you can also use the CLI to spin up and register a gateway in Consul. For additional information, refer to the [`consul connect envoy` command](/commands/connect/envoy#mesh-gateways). ### Sidecar registration -- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams` documentation](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) for details. +- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams` documentation](/docs/connect/registration/service-registration#upstream-configuration-reference) for details. - The service `proxy.upstreams.destination_name` is always required. - The `proxy.upstreams.destination_peer` must be configured to enable cross-cluster traffic. - The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. ### Service exports -- Include the `exported-services` configuration entry to enable Consul to export services contained in a cluster to one or more additional clusters. For additional information, refer to the [Exported Services documentation](/consul/docs/connect/config-entries/exported-services). +- Include the `exported-services` configuration entry to enable Consul to export services contained in a cluster to one or more additional clusters. For additional information, refer to the [Exported Services documentation](/docs/connect/config-entries/exported-services). ### ACL configuration -If ACLs are enabled, you must add a token granting `service:write` for the gateway's service name and `service:read` for all services in the Enterprise admin partition or OSS datacenter to the gateway's service definition. +If ACLs are enabled, you must add a token granting `service:write` for the gateway's service name and `service:read` for all services in the Enterprise admin partition or OSS datacenter to the gateway's service definition. + These permissions authorize the token to route communications for other Consul service mesh services. You must also grant `mesh:write` to mesh gateways routing peering traffic in the data plane. + This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. ### Modes Modes are configurable as either `remote` or `local` for mesh gateways that connect peered clusters. The `none` setting is invalid for mesh gateways in peered clusters and will be ignored by the gateway. -By default, all proxies connecting to peered clusters use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). +By default, all proxies connecting to peered clusters use mesh gateways in [remote mode](/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/create-manage-peering.mdx b/website/content/docs/connect/cluster-peering/create-manage-peering.mdx deleted file mode 100644 index dbbf0d4fc3ad..000000000000 --- a/website/content/docs/connect/cluster-peering/create-manage-peering.mdx +++ /dev/null @@ -1,537 +0,0 @@ ---- -layout: docs -page_title: Cluster Peering - Create and Manage Connections -description: >- - Generate a peering token to establish communication, export services, and authorize requests for cluster peering connections. Learn how to create, list, read, check, and delete peering connections. ---- - - -# Create and Manage Cluster Peering Connections - -A peering token enables cluster peering between different datacenters. Once you generate a peering token, you can use it to establish a connection between clusters. Then you can export services and create intentions so that peered clusters can call those services. - -## Create a peering connection - -Cluster peering is enabled by default on Consul servers as of v1.14. For additional information, including options to disable cluster peering, refer to [Configuration Files](/consul/docs/agent/config/config-files). - -The process to create a peering connection is a sequence with multiple steps: - -1. Create a peering token -1. Establish a connection between clusters -1. Export services between clusters -1. Authorize services for peers - -You can generate peering tokens and initiate connections on any available agent using either the API, CLI, or the Consul UI. If you use the API or CLI, we recommend performing these operations through a client agent in the partition you want to connect. - -The UI does not currently support exporting services between clusters or authorizing services for peers. - -### Create a peering token - -To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. - -Every time you generate a peering token, a single-use establishment secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. - - - - -In `cluster-01`, use the [`/peering/token` endpoint](/consul/api-docs/peering#generate-a-peering-token) to issue a request for a peering token. - -```shell-session -$ curl --request POST --data '{"Peer":"cluster-02"}' --url http://localhost:8500/v1/peering/token -``` - -The CLI outputs the peering token, which is a base64-encoded string containing the token details. - -Create a JSON file that contains the first cluster's name and the peering token. - - - -```json -{ - "Peer": "cluster-01", - "PeeringToken": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImF1ZCI6IlNvbHIifQ.5T7L_L1MPfQ_5FjKGa1fTPqrzwK4bNSM812nW6oyjb8" -} -``` - - - - - - -In `cluster-01`, use the [`consul peering generate-token` command](/consul/commands/peering/generate-token) to issue a request for a peering token. - -```shell-session -$ consul peering generate-token -name cluster-02 -``` - -The CLI outputs the peering token, which is a base64-encoded string containing the token details. -Save this value to a file or clipboard to be used in the next step on `cluster-02`. - - - - - -1. In the Consul UI for the datacenter associated with `cluster-01`, click **Peers**. -1. Click **Add peer connection**. -1. In the **Generate token** tab, enter `cluster-02` in the **Name of peer** field. -1. Click the **Generate token** button. -1. Copy the token before you proceed. You cannot view it again after leaving this screen. If you lose your token, you must generate a new one. - - - - -### Establish a connection between clusters - -Next, use the peering token to establish a secure connection between the clusters. - - - - -In one of the client agents in "cluster-02," use `peering_token.json` and the [`/peering/establish` endpoint](/consul/api-docs/peering#establish-a-peering-connection) to establish the peering connection. This endpoint does not generate an output unless there is an error. - -```shell-session -$ curl --request POST --data @peering_token.json http://127.0.0.1:8500/v1/peering/establish -``` - -When you connect server agents through cluster peering, their default behavior is to peer to the `default` partition. To establish peering connections for other partitions through server agents, you must add the `Partition` field to `peering_token.json` and specify the partitions you want to peer. For additional configuration information, refer to [Cluster Peering - HTTP API](/consul/api-docs/peering). - -You can dial the `peering/establish` endpoint once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. - - - - - -In one of the client agents in "cluster-02," issue the [`consul peering establish` command](/consul/commands/peering/establish) and specify the token generated in the previous step. The command establishes the peering connection. -The commands prints "Successfully established peering connection with cluster-01" after the connection is established. - -```shell-session -$ consul peering establish -name cluster-01 -peering-token token-from-generate -``` - -When you connect server agents through cluster peering, they peer their default partitions. -To establish peering connections for other partitions through server agents, you must add the `-partition` flag to the `establish` command and specify the partitions you want to peer. -For additional configuration information, refer to [`consul peering establish` command](/consul/commands/peering/establish) . - -You can run the `peering establish` command once per peering token. -Peering tokens cannot be reused after being used to establish a connection. -If you need to re-establish a connection, you must generate a new peering token. - - - - - -1. In the Consul UI for the datacenter associated with `cluster 02`, click **Peers** and then **Add peer connection**. -1. Click **Establish peering**. -1. In the **Name of peer** field, enter `cluster-01`. Then paste the peering token in the **Token** field. -1. Click **Add peer**. - - - - -### Export services between clusters - -After you establish a connection between the clusters, you need to create a configuration entry that defines the services that are available for other clusters. Consul uses this configuration entry to advertise service information and support service mesh connections across clusters. - -First, create a configuration entry and specify the `Kind` as `"exported-services"`. - - - -```hcl -Kind = "exported-services" -Name = "default" -Services = [ - { - ## The name and namespace of the service to export. - Name = "service-name" - Namespace = "default" - - ## The list of peer clusters to export the service to. - Consumers = [ - { - ## The peer name to reference in config is the one set - ## during the peering process. - Peer = "cluster-02" - } - ] - } -] -``` - - - -Then, add the configuration entry to your cluster. - -```shell-session -$ consul config write peering-config.hcl -``` - -Before you proceed, wait for the clusters to sync and make services available to their peers. You can issue an endpoint query to [check the peered cluster status](#check-peered-cluster-status). - -### Authorize services for peers - -Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. - -First, create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." The following example sets service intentions so that "frontend-service" can access "backend-service." - - - -```hcl -Kind = "service-intentions" -Name = "backend-service" - -Sources = [ - { - Name = "frontend-service" - Peer = "cluster-02" - Action = "allow" - } -] -``` - - - -If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. - -Then, add the configuration entry to your cluster. - -```shell-session -$ consul config write peering-intentions.hcl -``` - -### Authorize Service Reads with ACLs - -If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. -Read access to all imported services is granted using either of the following rules associated with an ACL token: -- `service:write` permissions for any service in the sidecar's partition. -- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. -For Consul Enterprise, access is granted to all imported services in the service's partition. -These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). - -Example rule files can be found in [Reading Servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` config entry documentation. - -Refer to [ACLs System Overview](/consul/docs/security/acl) for more information on ACLs and their setup. - -## Manage peering connections - -After you establish a peering connection, you can get a list of all active peering connections, read a specific peering connection's information, check peering connection health, and delete peering connections. - -### List all peering connections - -You can list all active peering connections in a cluster. - - - - -After you establish a peering connection, [query the `/peerings/` endpoint](/consul/api-docs/peering#list-all-peerings) to get a list of all peering connections. For example, the following command requests a list of all peering connections on `localhost` and returns the information as a series of JSON objects: - -```shell-session -$ curl http://127.0.0.1:8500/v1/peerings - -[ - { - "ID": "462c45e8-018e-f19d-85eb-1fc1bcc2ef12", - "Name": "cluster-02", - "State": "ACTIVE", - "Partition": "default", - "PeerID": "e83a315c-027e-bcb1-7c0c-a46650904a05", - "PeerServerName": "server.dc1.consul", - "PeerServerAddresses": [ - "10.0.0.1:8300" - ], - "CreateIndex": 89, - "ModifyIndex": 89 - }, - { - "ID": "1460ada9-26d2-f30d-3359-2968aa7dc47d", - "Name": "cluster-03", - "State": "INITIAL", - "Partition": "default", - "Meta": { - "env": "production" - }, - "CreateIndex": 109, - "ModifyIndex": 119 - }, -] -``` - - - - -After you establish a peering connection, run the [`consul peering list`](/consul/commands/peering/list) command to get a list of all peering connections. -For example, the following command requests a list of all peering connections and returns the information in a table: - -```shell-session -$ consul peering list - -Name State Imported Svcs Exported Svcs Meta -cluster-02 ACTIVE 0 2 env=production -cluster-03 PENDING 0 0 - ``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in a datacenter. - -The name that appears in the list is the name of the cluster in a different datacenter with an established peering connection. - - - -### Read a peering connection - -You can get information about individual peering connections between clusters. - - - - -After you establish a peering connection, [query the `/peering/` endpoint](/consul/api-docs/peering#read-a-peering-connection) to get peering information about for a specific cluster. For example, the following command requests peering connection information for "cluster-02" and returns the info as a JSON object: - -```shell-session -$ curl http://127.0.0.1:8500/v1/peering/cluster-02 - -{ - "ID": "462c45e8-018e-f19d-85eb-1fc1bcc2ef12", - "Name": "cluster-02", - "State": "INITIAL", - "PeerID": "e83a315c-027e-bcb1-7c0c-a46650904a05", - "PeerServerName": "server.dc1.consul", - "PeerServerAddresses": [ - "10.0.0.1:8300" - ], - "CreateIndex": 89, - "ModifyIndex": 89 -} -``` - - - - -After you establish a peering connection, run the [`consul peering read`](/consul/commands/peering/list) command to get peering information about for a specific cluster. -For example, the following command requests peering connection information for "cluster-02": - -```shell-session -$ consul peering read -name cluster-02 - -Name: cluster-02 -ID: 3b001063-8079-b1a6-764c-738af5a39a97 -State: ACTIVE -Meta: - env=production - -Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 -Peer Server Name: server.dc1.consul -Peer CA Pems: 0 -Peer Server Addresses: - 10.0.0.1:8300 - -Imported Services: 0 -Exported Services: 2 - -Create Index: 89 -Modify Index: 89 - -``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. Click the name of a peered cluster to view additional details about the peering connection. - - - -### Check peering connection health - -You can check the status of your peering connection to perform health checks. - -To confirm that the peering connection between your clusters remains healthy, query the [`health/service` endpoint](/consul/api-docs/health) of one cluster from the other cluster. For example, in "cluster-02," query the endpoint and add the `peer=cluster-01` query parameter to the end of the URL. - -```shell-session -$ curl \ - "http://127.0.0.1:8500/v1/health/service/?peer=cluster-01" -``` - -A successful query includes service information in the output. - -### Delete peering connections - -You can disconnect the peered clusters by deleting their connection. Deleting a peering connection stops data replication to the peer and deletes imported data, including services and CA certificates. - - - - -In "cluster-01," request the deletion through the [`/peering/ endpoint`](/consul/api-docs/peering#delete-a-peering-connection). - -```shell-session -$ curl --request DELETE http://127.0.0.1:8500/v1/peering/cluster-02 -``` - - - - -In "cluster-01," request the deletion through the [`consul peering delete`](/consul/commands/peering/list) command. - -```shell-session -$ consul peering delete -name cluster-02 - -Successfully submitted peering connection, cluster-02, for deletion -``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. - -Next to the name of the peer, click **More** (three horizontal dots) and then **Delete**. Click **Delete** to confirm and remove the peering connection. - - - - -## L7 traffic management between peers - -The following sections describe how to enable L7 traffic management features between peered clusters. - -### Service resolvers for redirects and failover - -As of Consul v1.14, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically failover and redirect between peers. The following examples update the [`service-resolver` config entry](/consul/docs/connect/config-entries/) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. - - - -```hcl -Kind = "service-resolver" -Name = "frontend" -ConnectTimeout = "15s" -Failover = { - "*" = { - Targets = [ - {Peer = "cluster-02"} - ] - } -} -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend -spec: - connectTimeout: 15s - failover: - '*': - targets: - - peer: 'cluster-02' - service: 'frontend' - namespace: 'default' -``` - -```json -{ - "ConnectTimeout": "15s", - "Kind": "service-resolver", - "Name": "frontend", - "Failover": { - "*": { - "Targets": [ - { - "Peer": "cluster-02" - } - ] - } - }, - "CreateIndex": 250, - "ModifyIndex": 250 -} -``` - - - -### Service splitters and custom routes - -The `service-splitter` and `service-router` configuration entry kinds do not support directly targeting a service instance hosted on a peer. To split or route traffic to a service on a peer, you must combine the definition with a `service-resolver` configuration entry that defines the service hosted on the peer as an upstream service. For example, to split traffic evenly between `frontend` services hosted on peers, first define the desired behavior locally: - - - -```hcl -Kind = "service-splitter" -Name = "frontend" -Splits = [ - { - Weight = 50 - ## defaults to service with same name as configuration entry ("frontend") - }, - { - Weight = 50 - Service = "frontend-peer" - }, -] -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: frontend -spec: - splits: - - weight: 50 - ## defaults to service with same name as configuration entry ("web") - - weight: 50 - service: frontend-peer -``` - -```json -{ - "Kind": "service-splitter", - "Name": "frontend", - "Splits": [ - { - "Weight": 50 - }, - { - "Weight": 50, - "Service": "frontend-peer" - } - ] -} -``` - - - -Then, create a local `service-resolver` configuration entry named `frontend-peer` and define a redirect targeting the peer and its service: - - - -```hcl -Kind = "service-resolver" -Name = "frontend-peer" -Redirect { - Service = frontend - Peer = "cluster-02" -} -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend-peer -spec: - redirect: - peer: 'cluster-02' - service: 'frontend' -``` - -```json -{ - "Kind": "service-resolver", - "Name": "frontend-peer", - "Redirect": { - "Service": "frontend", - "Peer": "cluster-02" - } -} -``` - - - diff --git a/website/content/docs/connect/cluster-peering/index.mdx b/website/content/docs/connect/cluster-peering/index.mdx index 77f4b8f4c733..aeb940d638c1 100644 --- a/website/content/docs/connect/cluster-peering/index.mdx +++ b/website/content/docs/connect/cluster-peering/index.mdx @@ -1,32 +1,37 @@ --- layout: docs -page_title: Service Mesh - What is Cluster Peering? +page_title: Cluster Peering Overview description: >- - Cluster peering establishes communication between independent clusters in Consul, allowing services to interact across datacenters. Learn about the cluster peering process, differences with WAN federation for multi-datacenter deployments, and technical constraints. + Cluster peering establishes communication between independent clusters in Consul, allowing services to interact across datacenters. Learn how cluster peering works, its differences with WAN federation for multi-datacenter deployments, and how to troubleshoot common issues. --- -# What is Cluster Peering? +# Cluster peering overview -You can create peering connections between two or more independent clusters so that services deployed to different partitions or datacenters can communicate. +This topic provides an overview of cluster peering, which lets you connect two or more independent Consul clusters so that services deployed to different partitions or datacenters can communicate. +Cluster peering is enabled in Consul by default. For specific information about cluster peering configuration and usage, refer to following pages. -## Overview +## What is cluster peering? -Cluster peering is a process that allows Consul clusters to communicate with each other. The cluster peering process consists of the following steps: +Consul supports cluster peering connections between two [admin partitions](/consul/docs/enterprise/admin-partitions) _in different datacenters_. Deployments without an Enterprise license can still use cluster peering because every datacenter automatically includes a default partition. Meanwhile, admin partitions _in the same datacenter_ do not require cluster peering connections because you can export services between them without generating or exchanging a peering token. -1. Create a peering token in one cluster. -1. Use the peering token to establish peering with a second cluster. -1. Export services between clusters. -1. Create intentions to authorize services for peers. +The following diagram describes Consul's cluster peering architecture. -This process establishes cluster peering between two [admin partitions](/consul/docs/enterprise/admin-partitions). Deployments without an Enterprise license can still use cluster peering because every datacenter automatically includes a `default` partition. +![Diagram of cluster peering with admin partitions](/img/cluster-peering-diagram.png) -For detailed instructions on establishing cluster peering connections, refer to [Create and Manage Peering Connections](/consul/docs/connect/cluster-peering/create-manage-peering). +In this diagram, the `default` partition in Consul DC 1 has a cluster peering connection with the `web` partition in Consul DC 2. Enforced by their respective mesh gateways, this cluster peering connection enables `Service B` to communicate with `Service C` as a service upstream. -> To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering-aws?utm_source=docs). +Cluster peering leverages several components of Consul's architecture to enforce secure communication between services: -### Differences between WAN federation and cluster peering +- A _peering token_ contains an embedded secret that securely establishes communication when shared symetrically between datacenters. Sharing this token enables each datacenter's server agents to recognize requests from authorized peers, similar to how the [gossip encryption key secures agent LAN gossip](/consul/docs/security/encryption#gossip-encryption). +- A _mesh gateway_ encrypts outgoing traffic, decrypts incoming traffic, and directs traffic to healthy services. Consul's service mesh features must be enabled in order to use mesh gateways. Mesh gateways support the specific admin partitions they are deployed on. Refer to [Mesh gateways](/consul/docs/connect/gateways/mesh-gateway) for more information. +- An _exported service_ communicates with downstreams deployed in other admin partitions. They are explicitly defined in an [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services). +- A _service intention_ secures [service-to-service communication in a service mesh](/consul/docs/connect/intentions). Intentions enable identity-based access between services by exchanging TLS certificates, which the service's sidecar proxy verifies upon each request. -WAN federation and cluster peering are different ways to connect Consul deployments. WAN federation connects multiple datacenters to make them function as if they were a single cluster, while cluster peering treats each datacenter as a separate cluster. As a result, WAN federation requires a primary datacenter to maintain and replicate global states such as ACLs and configuration entries, but cluster peering does not. +### Compared with WAN federation + +WAN federation and cluster peering are different ways to connect services through mesh gateways so that they can communicate across datacenters. WAN federation connects multiple datacenters to make them function as if they were a single cluster, while cluster peering treats each datacenter as a separate cluster. As a result, WAN federation requires a primary datacenter to maintain and replicate global states such as ACLs and configuration entries, but cluster peering does not. + +WAN federation and cluster peering also treat encrypted traffic differently. While mesh gateways between WAN federated datacenters use mTLS to keep data encrypted, mesh gateways between peers terminate mTLS sessions, decrypt data to HTTP services, and then re-encrypt traffic to send to services. Data must be decrypted in order to evaluate and apply dynamic routing rules at the destination cluster, which reduces coupling between peers. Regardless of whether you connect your clusters through WAN federation or cluster peering, human and machine users can use either method to discover services in other clusters or dial them through the service mesh. @@ -42,11 +47,40 @@ Regardless of whether you connect your clusters through WAN federation or cluste | Shares key/value stores | ✅ | ❌ | | Can replicate ACL tokens, policies, and roles | ✅ | ❌ | -## Important Cluster Peering Constraints +## Guidance + +The following resources are available to help you use Consul's cluster peering features. + +**Tutorials:** + +- To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) and Google Kubernetes Engine (GKE) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering). + +**Usage documentation:** + +- [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering) +- [Manage cluster peering connections](/consul/docs/connect/cluster-peering/usage/manage-connections) +- [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management) + +**Kubernetes-specific documentation:** + +- [Cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs) +- [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering) +- [Manage cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/manage-peering) +- [Manage L7 traffic with cluster peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/l7-traffic) + +**Reference documentation:** + +- [Cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs) +- [HTTP API reference: `/peering/` endpoint](/consul/api-docs/peering) +- [CLI reference: `peering` command](/consul/commands/peering). + +## Basic troubleshooting -Consider the following technical constraints: +If you experience errors when using Consul's cluster peering features, refer to the following list of technical constraints. +- Peer names can only contain lowercase characters. - Services with node, instance, and check definitions totaling more than 8MB cannot be exported to a peer. -- Two admin partitions in the same datacenter cannot be peered. Use [`exported-services`](/consul/docs/connect/config-entries/exported-services#exporting-services-to-peered-clusters) directly. -- The `consul intention` CLI command is not supported. To manage intentions that specify services in peered clusters, use [configuration entries](/consul/docs/connect/config-entries/service-intentions). -- Accessing key/value stores across peers is not supported. +- Two admin partitions in the same datacenter cannot be peered. Use the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services#exporting-services-to-peered-clusters) instead. +- To manage intentions that specify services in peered clusters, use [configuration entries](/consul/docs/connect/config-entries/service-intentions). The `consul intention` CLI command is not supported. +- The Consul UI does not support exporting services between clusters or creating service intentions. Use either the API or the CLI to complete these required steps when establishing new cluster peering connections. +- Accessing key/value stores across peers is not supported. \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/k8s.mdx b/website/content/docs/connect/cluster-peering/k8s.mdx deleted file mode 100644 index 570442fd4a72..000000000000 --- a/website/content/docs/connect/cluster-peering/k8s.mdx +++ /dev/null @@ -1,593 +0,0 @@ ---- -layout: docs -page_title: Cluster Peering on Kubernetes -description: >- - If you use Consul on Kubernetes, learn how to enable cluster peering, create peering CRDs, and then manage peering connections in consul-k8s. ---- - -# Cluster Peering on Kubernetes - -To establish a cluster peering connection on Kubernetes, you need to enable several pre-requisite values in the Helm chart and create custom resource definitions (CRDs) for each side of the peering. - -The following Helm values are mandatory for cluster peering: -- [`global.tls.enabled = true`](/consul/docs/k8s/helm#v-global-tls-enabled) -- [`meshGateway.enabled = true`](/consul/docs/k8s/helm#v-meshgateway-enabled) - -The following CRDs are used to create and manage a peering connection: - -- `PeeringAcceptor`: Generates a peering token and accepts an incoming peering connection. -- `PeeringDialer`: Uses a peering token to make an outbound peering connection with the cluster that generated the token. - -Peering connections, including both data plane and control plane traffic, is routed through mesh gateways. -As of Consul v1.14, you can also [implement service failovers and redirects to control traffic](/consul/docs/connect/l7-traffic) between peers. - -> To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering-aws?utm_source=docs). - -## Prerequisites - -You must implement the following requirements to create and use cluster peering connections with Kubernetes: - -- Consul v1.14.0 or later -- At least two Kubernetes clusters -- The installation must be running on Consul on Kubernetes version 1.0.0 or later - -### Prepare for installation - -Complete the following procedure after you have provisioned a Kubernetes cluster and set up your kubeconfig file to manage access to multiple Kubernetes clusters. - -1. Use the `kubectl` command to export the Kubernetes context names and then set them to variables for future use. For more information on how to use kubeconfig and contexts, refer to the [Kubernetes docs on configuring access to multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). - - You can use the following methods to get the context names for your clusters: - - - Use the `kubectl config current-context` command to get the context for the cluster you are currently in. - - Use the `kubectl config get-contexts` command to get all configured contexts in your kubeconfig file. - - ```shell-session - $ export CLUSTER1_CONTEXT= - $ export CLUSTER2_CONTEXT= - ``` - -1. To establish cluster peering through Kubernetes, create a `values.yaml` file with the following Helm values. **NOTE:** Mesh Gateway replicas are defaulted to 1 replica, and could be adjusted using the `meshGateway.replicas` value for higher availability. - - - - ```yaml - global: - name: consul - image: "hashicorp/consul:1.14.1" - peering: - enabled: true - tls: - enabled: true - meshGateway: - enabled: true - ``` - - - -### Install Consul on Kubernetes - -Install Consul on Kubernetes by using the CLI to apply `values.yaml` to each cluster. - - 1. In `cluster-01`, run the following commands: - - ```shell-session - $ export HELM_RELEASE_NAME=cluster-01 - ``` - - ```shell-session - $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc1 --kube-context $CLUSTER1_CONTEXT - ``` - - 1. In `cluster-02`, run the following commands: - - ```shell-session - $ export HELM_RELEASE_NAME=cluster-02 - ``` - - ```shell-session - $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc2 --kube-context $CLUSTER2_CONTEXT - ``` - -## Create a peering connection for Consul on Kubernetes - -To peer Kubernetes clusters running Consul, you need to create a peering token on one cluster (`cluster-01`) and share -it with the other cluster (`cluster-02`). The generated peering token from `cluster-01` will include the addresses of -the servers for that cluster. The servers for `cluster-02` will use that information to dial the servers in -`cluster-01`. Complete the following steps to create the peer connection. - -### Using mesh gateways for the peering connection -If the servers in `cluster-01` are not directly routable from the dialing cluster `cluster-02`, then you'll need to set up peering through mesh gateways. - -1. In `cluster-01` apply the `Mesh` custom resource so the generated token will have the mesh gateway addresses which will be routable from the other cluster. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: Mesh - metadata: - name: mesh - spec: - peering: - peerThroughMeshGateways: true - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply -f mesh.yaml - ``` - -1. In `cluster-02` apply the `Mesh` custom resource so that the servers for `cluster-02` will use their local mesh gateway to dial the servers for `cluster-01`. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: Mesh - metadata: - name: mesh - spec: - peering: - peerThroughMeshGateways: true - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply -f mesh.yaml - ``` - -### Create a peering token - -Peers identify each other using the `metadata.name` values you establish when creating the `PeeringAcceptor` and `PeeringDialer` CRDs. - -1. In `cluster-01`, create the `PeeringAcceptor` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringAcceptor - metadata: - name: cluster-02 ## The name of the peer you want to connect to - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. Apply the `PeeringAcceptor` resource to the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply --filename acceptor.yaml - ```` - -1. Save your peering token so that you can export it to the other cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT get secret peering-token --output yaml > peering-token.yaml - ``` - -### Establish a peering connection between clusters - -1. Apply the peering token to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename peering-token.yaml - ``` - -1. In `cluster-02`, create the `PeeringDialer` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringDialer - metadata: - name: cluster-01 ## The name of the peer you want to connect to - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. Apply the `PeeringDialer` resource to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename dialer.yaml - ``` - -### Configure the mesh gateway mode for traffic between services -Mesh gateways are required for service-to-service traffic between peered clusters. By default, this will mean that a -service dialing another service in a remote peer will dial the remote mesh gateway to reach that service. If you would -like to configure the mesh gateway mode such that this traffic always leaves through the local mesh gateway, you can use the `ProxyDefaults` CRD. - -1. In `cluster-01` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ProxyDefaults - metadata: - name: global - spec: - meshGateway: - mode: local - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply -f proxy-defaults.yaml - ``` - -1. In `cluster-02` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ProxyDefaults - metadata: - name: global - spec: - meshGateway: - mode: local - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply -f proxy-defaults.yaml - ``` - -### Export services between clusters - -The examples described in this section demonstrate how to export a service named `backend`. You should change instances of `backend` in the example code to the name of the service you want to export. - -1. For the service in `cluster-02` that you want to export, add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods prior to deploying. The annotation allows the workload to join the mesh. It is highlighted in the following example: - - - - ```yaml - # Service to expose backend - apiVersion: v1 - kind: Service - metadata: - name: backend - spec: - selector: - app: backend - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 9090 - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: backend - --- - # Deployment for backend - apiVersion: apps/v1 - kind: Deployment - metadata: - name: backend - labels: - app: backend - spec: - replicas: 1 - selector: - matchLabels: - app: backend - template: - metadata: - labels: - app: backend - annotations: - "consul.hashicorp.com/connect-inject": "true" - spec: - serviceAccountName: backend - containers: - - name: backend - image: nicholasjackson/fake-service:v0.22.4 - ports: - - containerPort: 9090 - env: - - name: "LISTEN_ADDR" - value: "0.0.0.0:9090" - - name: "NAME" - value: "backend" - - name: "MESSAGE" - value: "Response from backend" - ``` - - - -1. Deploy the `backend` service to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename backend.yaml - ``` - -1. In `cluster-02`, create an `ExportedServices` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ExportedServices - metadata: - name: default ## The name of the partition containing the service - spec: - services: - - name: backend ## The name of the service you want to export - consumers: - - peer: cluster-01 ## The name of the peer that receives the service - ``` - - - -1. Apply the `ExportedServices` resource to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename exported-service.yaml - ``` - -### Authorize services for peers - -1. Create service intentions for the second cluster. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ServiceIntentions - metadata: - name: backend-deny - spec: - destination: - name: backend - sources: - - name: "*" - action: deny - - name: frontend - action: allow - peer: cluster-01 ## The peer of the source service - ``` - - - -1. Apply the intentions to the second cluster. - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yaml - ``` - - - -1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. - - - - ```yaml - # Service to expose frontend - apiVersion: v1 - kind: Service - metadata: - name: frontend - spec: - selector: - app: frontend - ports: - - name: http - protocol: TCP - port: 9090 - targetPort: 9090 - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: frontend - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: frontend - labels: - app: frontend - spec: - replicas: 1 - selector: - matchLabels: - app: frontend - template: - metadata: - labels: - app: frontend - annotations: - "consul.hashicorp.com/connect-inject": "true" - spec: - serviceAccountName: frontend - containers: - - name: frontend - image: nicholasjackson/fake-service:v0.22.4 - securityContext: - capabilities: - add: ["NET_ADMIN"] - ports: - - containerPort: 9090 - env: - - name: "LISTEN_ADDR" - value: "0.0.0.0:9090" - - name: "UPSTREAM_URIS" - value: "http://backend.virtual.cluster-02.consul" - - name: "NAME" - value: "frontend" - - name: "MESSAGE" - value: "Hello World" - - name: "HTTP_CLIENT_KEEP_ALIVES" - value: "false" - ``` - - - -1. Apply the service file to the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply --filename frontend.yaml - ``` - -1. Run the following command in `frontend` and then check the output to confirm that you peered your clusters successfully. - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090 - - { - "name": "frontend", - "uri": "/", - "type": "HTTP", - "ip_addresses": [ - "10.16.2.11" - ], - "start_time": "2022-08-26T23:40:01.167199", - "end_time": "2022-08-26T23:40:01.226951", - "duration": "59.752279ms", - "body": "Hello World", - "upstream_calls": { - "http://backend.virtual.cluster-02.consul": { - "name": "backend", - "uri": "http://backend.virtual.cluster-02.consul", - "type": "HTTP", - "ip_addresses": [ - "10.32.2.10" - ], - "start_time": "2022-08-26T23:40:01.223503", - "end_time": "2022-08-26T23:40:01.224653", - "duration": "1.149666ms", - "headers": { - "Content-Length": "266", - "Content-Type": "text/plain; charset=utf-8", - "Date": "Fri, 26 Aug 2022 23:40:01 GMT" - }, - "body": "Response from backend", - "code": 200 - } - }, - "code": 200 - } - ``` - - - -## End a peering connection - -To end a peering connection, delete both the `PeeringAcceptor` and `PeeringDialer` resources. - -1. Delete the `PeeringDialer` resource from the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT delete --filename dialer.yaml - ``` - -1. Delete the `PeeringAcceptor` resource from the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT delete --filename acceptor.yaml - ```` - -1. Confirm that you deleted your peering connection in `cluster-01` by querying the the `/health` HTTP endpoint. The peered services should no longer appear. - - 1. Exec into the server pod for the first cluster. - - ```shell-session - $ kubectl exec -it consul-server-0 --context $CLUSTER1_CONTEXT -- /bin/sh - ``` - - 1. If you've enabled ACLs, export an ACL token to access the `/health` HTP endpoint for services. The bootstrap token may be used if an ACL token is not already provisioned. - - ```shell-session - $ export CONSUL_HTTP_TOKEN= - ``` - - 1. Query the the `/health` HTTP endpoint. The peered services should no longer appear. - - ```shell-session - $ curl "localhost:8500/v1/health/connect/backend?peer=cluster-02" - ``` - -## Recreate or reset a peering connection - -To recreate or reset the peering connection, you need to generate a new peering token from the cluster where you created the `PeeringAcceptor`. - -1. In the `PeeringAcceptor` CRD, add the annotation `consul.hashicorp.com/peering-version`. If the annotation already exists, update its value to a higher version. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringAcceptor - metadata: - name: cluster-02 - annotations: - consul.hashicorp.com/peering-version: "1" ## The peering version you want to set, must be in quotes - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. After updating `PeeringAcceptor`, repeat the following steps to create a peering connection: - 1. [Create a peering token](#create-a-peering-token) - 1. [Establish a peering connection between clusters](#establish-a-peering-connection-between-clusters) - 1. [Export services between clusters](#export-services-between-clusters) - 1. [Authorize services for peers](#authorize-services-for-peers) - - Your peering connection is re-established with the updated token. - -~> **Note:** The only way to create or set a new peering token is to manually adjust the value of the annotation `consul.hashicorp.com/peering-version`. Creating a new token causes the previous token to expire. - -## Traffic management between peers - -As of Consul v1.14, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically failover and redirect between peers. - -To configure automatic service failovers and redirect, edit the `ServiceResolver` CRD so that traffic resolves to a backup service instance on a peer. The following example updates the `ServiceResolver` CRD in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in `cluster-02` when it detects multiple connection failures to the primary instance. - - - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend -spec: - connectTimeout: 15s - failover: - '*': - targets: - - peer: 'cluster-02' - service: 'backup' - namespace: 'default' -``` - - diff --git a/website/content/docs/connect/cluster-peering/tech-specs.mdx b/website/content/docs/connect/cluster-peering/tech-specs.mdx new file mode 100644 index 000000000000..3e00d6a48b55 --- /dev/null +++ b/website/content/docs/connect/cluster-peering/tech-specs.mdx @@ -0,0 +1,69 @@ +--- +layout: docs +page_title: Cluster Peering Technical Specifications +description: >- + Cluster peering connections in Consul interact with mesh gateways, sidecar proxies, exported services, and ACLs. Learn about the configuration requirements for these components. +--- + +# Cluster peering technical specifications + +This reference topic describes the technical specifications associated with using cluster peering in your deployments. These specifications include required Consul components and their configurations. + +## Requirements + +To use cluster peering features, make sure your Consul environment meets the following prerequisites: + +- Consul v1.14 or higher. +- A local Consul agent is required to manage mesh gateway configuration. +- Use [Envoy proxies](/consul/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. + +In addition, the following service mesh components are required in order to establish cluster peering connections: + +- [Mesh gateways](#mesh-gateway-requirements) +- [Sidecar proxies](#sidecar-proxy-requirements) +- [Exported services](#exported-service-requirements) +- [ACLs](#acl-requirements) + +### Mesh gateway requirements + +Mesh gateways are required for routing service mesh traffic between partitions with cluster peering connections. Consider the following general requirements for mesh gateways when using cluster peering: + +- A cluster requires a registered mesh gateway in order to export services to peers. +- For Enterprise, this mesh gateway must also be registered in the same partition as the exported services and their `exported-services` configuration entry. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. + +In addition, you must define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. + +#### Mesh gateway modes + +By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. + +- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. +- The `none` mode is invalid for mesh gateways with cluster peering connections. + +Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) for more information. + +## Sidecar proxy requirements + +The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/discovery/services). + +- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams`](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) documentation for details. +- The `proxy.upstreams.destination_name` parameter is always required. +- The `proxy.upstreams.destination_peer` parameter must be configured to enable cross-cluster traffic. +- The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. + +## Exported service requirements + +The `exported-services` configuration entry is required in order for services to communicate across partitions with cluster peering connections. + +Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/create-cluster-peering). + +Refer to the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) reference for more information. + +## ACL requirements + +If ACLs are enabled, you must add tokens to grant the following permissions: + +- Grant `service:write` permissions to services that define mesh gateways in their server definition. +- Grant `service:read` permissions for all services on the partition. +- Grant `mesh:write` permissions to the mesh gateways that participate in cluster peering connections. This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx new file mode 100644 index 000000000000..29259ccf2e57 --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx @@ -0,0 +1,271 @@ +--- +layout: docs +page_title: Establish Cluster Peering Connections +description: >- + Generate a peering token to establish communication, export services, and authorize requests for cluster peering connections. Learn how to establish peering connections with Consul's HTTP API, CLI or UI. +--- + +# Establish cluster peering connections + +This page details the process for establishing a cluster peering connection between services deployed to different datacenters. You can interact with Consul's cluster peering features using the CLI, the HTTP API, or the UI. The overall process for establishing a cluster peering connection consists of the following steps: + +1. Create a peering token in one cluster. +1. Use the peering token to establish peering with a second cluster. +1. Export services between clusters. +1. Create intentions to authorize services for peers. + +Cluster peering between services cannot be established until all four steps are complete. + +For Kubernetes-specific guidance for establishing cluster peering connections, refer to [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). + +## Requirements + +You must meet the following requirements to use cluster peering: + +- Consul v1.14.1 or higher +- Services hosted in admin partitions on separate datacenters + +If you need to make services available to an admin partition in the same datacenter, do not use cluster peering. Instead, use the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) to make service upstreams available to other admin partitions in a single datacenter. + +### Mesh gateway requirements + +Mesh gateways are required for all cluster peering connections. Consider the following architectural requirements when creating mesh gateways: + +- A registered mesh gateway is required in order to export services to peers. +- For Enterprise, the mesh gateway that exports services must be registered in the same partition as the exported services and their `exported-services` configuration entry. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. + +## Create a peering token + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + + + + +1. In `cluster-01`, use the [`consul peering generate-token` command](/consul/commands/peering/generate-token) to issue a request for a peering token. + + ```shell-session + $ consul peering generate-token -name cluster-02 + ``` + + The CLI outputs the peering token, which is a base64-encoded string containing the token details. + +1. Save this value to a file or clipboard to use in the next step on `cluster-02`. + + + + +1. In `cluster-01`, use the [`/peering/token` endpoint](/consul/api-docs/peering#generate-a-peering-token) to issue a request for a peering token. + + ```shell-session + $ curl --request POST --data '{"Peer":"cluster-02"}' --url http://localhost:8500/v1/peering/token + ``` + + The CLI outputs the peering token, which is a base64-encoded string containing the token details. + +1. Create a JSON file that contains the first cluster's name and the peering token. + + + + ```json + { + "Peer": "cluster-01", + "PeeringToken": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImF1ZCI6IlNvbHIifQ.5T7L_L1MPfQ_5FjKGa1fTPqrzwK4bNSM812nW6oyjb8" + } + ``` + + + + + + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + +1. In the Consul UI for the datacenter associated with `cluster-01`, click **Peers**. +1. Click **Add peer connection**. +1. In the **Generate token** tab, enter `cluster-02` in the **Name of peer** field. +1. Click the **Generate token** button. +1. Copy the token before you proceed. You cannot view it again after leaving this screen. If you lose your token, you must generate a new one. + + + + +## Establish a connection between clusters + +Next, use the peering token to establish a secure connection between the clusters. + + + + +1. In one of the client agents deployed to "cluster-02," issue the [`consul peering establish` command](/consul/commands/peering/establish) and specify the token generated in the previous step. + + ```shell-session + $ consul peering establish -name cluster-01 -peering-token token-from-generate + "Successfully established peering connection with cluster-01" + ``` + +When you connect server agents through cluster peering, they peer their default partitions. To establish peering connections for other partitions through server agents, you must add the `-partition` flag to the `establish` command and specify the partitions you want to peer. For additional configuration information, refer to [`consul peering establish` command](/consul/commands/peering/establish). + +You can run the `peering establish` command once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. + + + + +1. In one of the client agents in "cluster-02," use `peering_token.json` and the [`/peering/establish` endpoint](/consul/api-docs/peering#establish-a-peering-connection) to establish the peering connection. This endpoint does not generate an output unless there is an error. + + ```shell-session + $ curl --request POST --data @peering_token.json http://127.0.0.1:8500/v1/peering/establish + ``` + +When you connect server agents through cluster peering, their default behavior is to peer to the `default` partition. To establish peering connections for other partitions through server agents, you must add the `Partition` field to `peering_token.json` and specify the partitions you want to peer. For additional configuration information, refer to [Cluster Peering - HTTP API](/consul/api-docs/peering). + +You can dial the `peering/establish` endpoint once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. + + + + + +1. In the Consul UI for the datacenter associated with `cluster 02`, click **Peers** and then **Add peer connection**. +1. Click **Establish peering**. +1. In the **Name of peer** field, enter `cluster-01`. Then paste the peering token in the **Token** field. +1. Click **Add peer**. + + + + +## Export services between clusters + +After you establish a connection between the clusters, you need to create an `exported-services` configuration entry that defines the services that are available for other clusters. Consul uses this configuration entry to advertise service information and support service mesh connections across clusters. + +An `exported-services` configuration entry makes services available to another admin partition. While it can target admin partitions either locally or remotely. Clusters peers always export services to remote partitions. Refer to [exported service consumers](/consul/docs/connect/config-entries/exported-services#consumers-1) for more information. + +You must use the Consul CLI to complete this step. The HTTP API and the Consul UI do not support `exported-services` configuration entries. + + + + +1. Create a configuration entry and specify the `Kind` as `"exported-services"`. + + + + ```hcl + Kind = "exported-services" + Name = "default" + Services = [ + { + ## The name and namespace of the service to export. + Name = "service-name" + Namespace = "default" + + ## The list of peer clusters to export the service to. + Consumers = [ + { + ## The peer name to reference in config is the one set + ## during the peering process. + Peer = "cluster-02" + } + ] + } + ] + ``` + + + +1. Add the configuration entry to your cluster. + + ```shell-session + $ consul config write peering-config.hcl + ``` + +Before you proceed, wait for the clusters to sync and make services available to their peers. To check the peered cluster status, [read the cluster peering connection](/consul/docs/connect/cluster-peering/usage/manage-connections#read-a-peering-connection). + + + + +## Authorize services for peers + +Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. + +You must use the HTTP API or the Consul CLI to complete this step. The Consul UI supports intentions for local clusters only. + + + + +1. Create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: + + + + ```hcl + Kind = "service-intentions" + Name = "backend-service" + + Sources = [ + { + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` + + + + If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. + +1. Add the configuration entry to your cluster. + + ```shell-session + $ consul config write peering-intentions.hcl + ``` + + + + +1. Create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: + + + + ```hcl + Kind = "service-intentions" + Name = "backend-service" + + Sources = [ + { + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` + + + + If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. + +1. Add the configuration entry to your cluster. + + ```shell-session + $ curl --request PUT --data @peering-intentions.hcl http://127.0.0.1:8500/v1/config + ``` + + + + +### Authorize service reads with ACLs + +If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. + +Read access to all imported services is granted using either of the following rules associated with an ACL token: + +- `service:write` permissions for any service in the sidecar's partition. +- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. + +For Consul Enterprise, the permissions apply to all imported services in the service's partition. These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). + +Refer to [Reading servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` configuration entry documentation for example rules. + +For additional information about how to configure and use ACLs, refer to [ACLs system overview](/consul/docs/security/acl). \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx new file mode 100644 index 000000000000..d2e3b77181fc --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx @@ -0,0 +1,137 @@ +--- +layout: docs +page_title: Manage Cluster Peering Connections +description: >- + Learn how to list, read, and delete cluster peering connections using Consul. You can use the HTTP API, the CLI, or the Consul UI to manage cluster peering connections. +--- + +# Manage cluster peering connections + +This usage topic describes how to manage cluster peering connections using the CLI, the HTTP API, and the UI. + +After you establish a cluster peering connection, you can get a list of all active peering connections, read a specific peering connection's information, and delete peering connections. + +For Kubernetes-specific guidance for managing cluster peering connections, refer to [Manage cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/manage-peering). + +## List all peering connections + +You can list all active peering connections in a cluster. + + + + + ```shell-session + $ consul peering list + Name State Imported Svcs Exported Svcs Meta + cluster-02 ACTIVE 0 2 env=production + cluster-03 PENDING 0 0 + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering list` CLI command reference](/consul/commands/peering/list). + + + + +The following example shows how to format an API request to list peering connections: + + ```shell-session + $ curl --header "X-Consul-Token: 0137db51-5895-4c25-b6cd-d9ed992f4a52" http://127.0.0.1:8500/v1/peerings + ``` + +For more information, including optional parameters and sample responses, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#list-all-peerings). + + + + +In the Consul UI, click **Peers**. + +The UI lists peering connections you created for clusters in a datacenter. The name that appears in the list is the name of the cluster in a different datacenter with an established peering connection. + + + + +## Read a peering connection + +You can get information about individual peering connections between clusters. + + + + + +The following example outputs information about a peering connection locally referred to as "cluster-02": + + ```shell-session + $ consul peering read -name cluster-02 + Name: cluster-02 + ID: 3b001063-8079-b1a6-764c-738af5a39a97 + State: ACTIVE + Meta: + env=production + + Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 + Peer Server Name: server.dc1.consul + Peer CA Pems: 0 + Peer Server Addresses: + 10.0.0.1:8300 + + Imported Services: 0 + Exported Services: 2 + + Create Index: 89 + Modify Index: 89 + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering read` CLI command reference](/consul/commands/peering/read). + + + + + ```shell-session + $ curl --header "X-Consul-Token: b23b3cad-5ea1-4413-919e-c76884b9ad60" http://127.0.0.1:8500/v1/peering/cluster-02 + ``` + +For more information, including optional parameters and sample responses, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#read-a-peering-connection). + + + + +1. In the Consul UI, click **Peers**. + +1. Click the name of a peered cluster to view additional details about the peering connection. + + + + +## Delete peering connections + +You can disconnect the peered clusters by deleting their connection. Deleting a peering connection stops data replication to the peer and deletes imported data, including services and CA certificates. + + + + + The following examples deletes a peering connection to a cluster locally referred to as "cluster-02": + + ```shell-session + $ consul peering delete -name cluster-02 + Successfully submitted peering connection, cluster-02, for deletion + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering delete` CLI command reference](/consul/commands/peering/delete). + + + + + ```shell-session + $ curl --request DELETE --header "X-Consul-Token: b23b3cad-5ea1-4413-919e-c76884b9ad60" http://127.0.0.1:8500/v1/peering/cluster-02 + ``` + +This endpoint does not return a response. For more information, including optional parameters, refer to the [`/peering` endpoint reference](/consul/api-docs/peering/consul/api-docs/peering#delete-a-peering-connection). + + + +1. In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. +1. Next to the name of the peer, click **More** (three horizontal dots) and then **Delete**. +1. Click **Delete** to confirm and remove the peering connection. + + + \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx b/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx new file mode 100644 index 000000000000..240b56dc9742 --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx @@ -0,0 +1,168 @@ +--- +layout: docs +page_title: Cluster Peering L7 Traffic Management +description: >- + Combine service resolver configurations with splitter and router configurations to manage L7 traffic in Consul deployments with cluster peering connections. Learn how to define dynamic traffic rules to target peers for redirects and failover. +--- + +# Manage L7 traffic with cluster peering + +This usage topic describes how to configure and apply the [`service-resolver` configuration entry](/consul/docs/connect/config-entries/service-resolver) to set up redirects and failovers between services that have an existing cluster peering connection. + +For Kubernetes-specific guidance for managing L7 traffic with cluster peering, refer to [Manage L7 traffic with cluster peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/l7-traffic). + +## Service resolvers for redirects and failover + +When you use cluster peering to connect datacenters through their admin partitions, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically forward traffic to services hosted on peer clusters. + +However, the `service-splitter` and `service-router` configuration entry kinds do not natively support directly targeting a service instance hosted on a peer. Before you can split or route traffic to a service on a peer, you must define the service hosted on the peer as an upstream service by configuring a failover in the `service-resolver` configuration entry. Then, you can set up a redirect in a second service resolver to interact with the peer service by name. + +For more information about formatting, updating, and managing configuration entries in Consul, refer to [How to use configuration entries](/consul/docs/agent/config-entries). + +## Configure dynamic traffic between peers + +To configure L7 traffic management behavior in deployments with cluster peering connections, complete the following steps in order: + +1. Define the peer cluster as a failover target in the service resolver configuration. + + The following examples update the [`service-resolver` configuration entry](/consul/docs/connect/config-entries/service-resolver) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. + + + + ```hcl + Kind = "service-resolver" + Name = "frontend" + ConnectTimeout = "15s" + Failover = { + "*" = { + Targets = [ + {Peer = "cluster-02"} + ] + } + } + ``` + + ```json + { + "ConnectTimeout": "15s", + "Kind": "service-resolver", + "Name": "frontend", + "Failover": { + "*": { + "Targets": [ + { + "Peer": "cluster-02" + } + ] + } + }, + "CreateIndex": 250, + "ModifyIndex": 250 + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend + spec: + connectTimeout: 15s + failover: + '*': + targets: + - peer: 'cluster-02' + service: 'frontend' + namespace: 'default' + ``` + + + +1. Define the desired behavior in `service-splitter` or `service-router` configuration entries. + + The following example splits traffic evenly between `frontend` services hosted on peers by defining the desired behavior locally: + + + + ```hcl + Kind = "service-splitter" + Name = "frontend" + Splits = [ + { + Weight = 50 + ## defaults to service with same name as configuration entry ("frontend") + }, + { + Weight = 50 + Service = "frontend-peer" + }, + ] + ``` + + ```json + { + "Kind": "service-splitter", + "Name": "frontend", + "Splits": [ + { + "Weight": 50 + }, + { + "Weight": 50, + "Service": "frontend-peer" + } + ] + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceSplitter + metadata: + name: frontend + spec: + splits: + - weight: 50 + ## defaults to service with same name as configuration entry ("frontend") + - weight: 50 + service: frontend-peer + ``` + + + +1. Create a local `service-resolver` configuration entry named `frontend-peer` and define a redirect targeting the peer and its service: + + + + ```hcl + Kind = "service-resolver" + Name = "frontend-peer" + Redirect { + Service = frontend + Peer = "cluster-02" + } + ``` + + ```json + { + "Kind": "service-resolver", + "Name": "frontend-peer", + "Redirect": { + "Service": "frontend", + "Peer": "cluster-02" + } + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend-peer + spec: + redirect: + peer: 'cluster-02' + service: 'frontend' + ``` + + \ No newline at end of file diff --git a/website/content/docs/connect/config-entries/service-intentions.mdx b/website/content/docs/connect/config-entries/service-intentions.mdx index 619b4fb9fed3..34c948ac1efd 100644 --- a/website/content/docs/connect/config-entries/service-intentions.mdx +++ b/website/content/docs/connect/config-entries/service-intentions.mdx @@ -335,6 +335,57 @@ spec: ``` +### Cluster peering + +When using cluster peering connections, intentions secure your deployments with authorized service-to-service communication between remote datacenters. In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: + + + + ```hcl + Kind = "service-intentions" + Name = "backend-service" + + Sources = [ + { + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` + + ```yaml + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceIntentions + metadata: + name: backend-deny + spec: + destination: + name: backend + sources: + - name: "*" + action: deny + - name: frontend + action: allow + peer: cluster-01 ## The peer of the source service + ``` + + ```json + { + "Kind": "service-intentions", + "Name": "backend-service", + "Sources": [ + { + "Name": "frontend-service", + "Peer": "cluster-02", + "Action": "allow" + } + ] + } + ``` + + ## Available Fields - + In Kubernetes deployments, cluster peering connections interact with mesh gateways, sidecar proxies, exported services, and ACLs. Learn about requirements specific to k8s, including required Helm values and CRDs. +--- + +# Cluster peering on Kubernetes technical specifications + +This reference topic describes the technical specifications associated with using cluster peering in your Kubernetes deployments. These specifications include [required Helm values](#helm-requirements) and [required custom resource definitions (CRDs)](#crd-requirements), as well as required Consul components and their configurations. + +For cluster peering requirements in non-Kubernetes deployments, refer to [cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs). + +## General Requirements + +To use cluster peering features, make sure your Consul environment meets the following prerequisites: + +- Consul v1.14 or higher +- Consul on Kubernetes v1.0.0 or higher +- At least two Kubernetes clusters + +In addition, the following service mesh components are required in order to establish cluster peering connections: + +- [Helm](#helm-requirements) +- [Custom resource definitions (CRD)](#crd-requirements) +- [Mesh gateways](#mesh-gateway-requirements) +- [Exported services](#exported-service-requirements) +- [ACLs](#acl-requirements) + +## Helm requirements + + Mesh gateways are required when establishing cluster peering connections. The following values must be set in the Helm chart to enable mesh gateways: + +- [`global.tls.enabled = true`](/consul/docs/k8s/helm#v-global-tls-enabled) +- [`meshGateway.enabled = true`](/consul/docs/k8s/helm#v-meshgateway-enabled) + +Refer to the following example Helm configuration: + + + +```yaml +global: + name: consul + image: "hashicorp/consul:1.14.1" + peering: + enabled: true + tls: + enabled: true +meshGateway: + enabled: true +``` + + + +After mesh gateways are enabled in the Helm chart, you can separately [configure Mesh CRDs](#mesh-gateway-configuration-for-kubernetes). + +## CRD requirements + +You must create the following CRDs in order to establish a peering connection: + +- `PeeringAcceptor`: Generates a peering token and accepts an incoming peering connection. +- `PeeringDialer`: Uses a peering token to make an outbound peering connection with the cluster that generated the token. + +Refer to the following example CRDs: + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: PeeringAcceptor +metadata: + name: cluster-02 ## The name of the peer you want to connect to +spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: PeeringDialer +metadata: + name: cluster-01 ## The name of the peer you want to connect to +spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" +``` + + + + +## Mesh gateway requirements + +Mesh gateways are required for routing service mesh traffic between partitions with cluster peering connections. Consider the following general requirements for mesh gateways when using cluster peering: + +- A cluster requires a registered mesh gateway in order to export services to peers. +- For Enterprise, this mesh gateway must also be registered in the same partition as the exported services and their `exported-services` configuration entry. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. + +In addition, you must define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. + +### Mesh gateway modes + +By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. + +- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. +- The `none` mode is invalid for mesh gateways with cluster peering connections. + +Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) for more information. + +### Mesh gateway configuration for Kubernetes + +Mesh gateways are required for cluster peering connections. Complete the following steps to add mesh gateways to your deployment so that you can establish cluster peering connections: + +1. In `cluster-01` create the `Mesh` custom resource with `peeringThroughMeshGateways` set to `true`. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: Mesh + metadata: + name: mesh + spec: + peering: + peerThroughMeshGateways: true + ``` + + + +1. Apply the mesh gateway to `cluster-01`. Replace `CLUSTER1_CONTEXT` to target the first Consul cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply -f mesh.yaml + ``` + +1. Repeat the process to create and apply a mesh gateway with cluster peering enabled to `cluster-02`. Replace `CLUSTER2_CONTEXT` to target the second Consul cluster. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: Mesh + metadata: + name: mesh + spec: + peering: + peerThroughMeshGateways: true + ``` + + + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply -f mesh.yaml + ``` + +## Exported service requirements + +The `exported-services` CRD is required in order for services to communicate across partitions with cluster peering connections. + +Refer to the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) reference for more information. + +## ACL requirements + +If ACLs are enabled, you must add tokens to grant the following permissions: + +- Grant `service:write` permissions to services that define mesh gateways in their server definition. +- Grant `service:read` permissions for all services on the partition. +- Grant `mesh:write` permissions to the mesh gateways that participate in cluster peering connections. This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx new file mode 100644 index 000000000000..71e0d46638ee --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx @@ -0,0 +1,453 @@ +--- +layout: docs +page_title: Establish Cluster Peering Connections on Kubernetes +description: >- + To establish a cluster peering connection on Kubernetes, generate a peering token to establish communication. Then export services and authorize requests with service intentions. +--- + +# Establish cluster peering connections on Kubernetes + +This page details the process for establishing a cluster peering connection between services in a Consul on Kubernetes deployment. + +The overall process for establishing a cluster peering connection consists of the following steps: + +1. Create a peering token in one cluster. +1. Use the peering token to establish peering with a second cluster. +1. Export services between clusters. +1. Create intentions to authorize services for peers. + +Cluster peering between services cannot be established until all four steps are complete. + +For general guidance for establishing cluster peering connections, refer to [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering). + +## Prerequisites + +You must meet the following requirements to use Consul's cluster peering features with Kubernetes: + +- Consul v1.14.1 or higher +- Consul on Kubernetes v1.0.0 or higher +- At least two Kubernetes clusters + +In Consul on Kubernetes, peers identify each other using the `metadata.name` values you establish when creating the `PeeringAcceptor` and `PeeringDialer` CRDs. For additional information about requirements for cluster peering on Kubernetes deployments, refer to [Cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs). + +### Assign cluster IDs to environmental variables + +After you provision a Kubernetes cluster and set up your kubeconfig file to manage access to multiple Kubernetes clusters, you can assign your clusters to environmental variables for future use. + +1. Get the context names for your Kubernetes clusters using one of these methods: + + - Run the `kubectl config current-context` command to get the context for the cluster you are currently in. + - Run the `kubectl config get-contexts` command to get all configured contexts in your kubeconfig file. + +1. Use the `kubectl` command to export the Kubernetes context names and then set them to variables. For more information on how to use kubeconfig and contexts, refer to the [Kubernetes docs on configuring access to multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). + + ```shell-session + $ export CLUSTER1_CONTEXT= + $ export CLUSTER2_CONTEXT= + ``` + +### Update the Helm chart + +To use cluster peering with Consul on Kubernetes deployments, update the Helm chart with [the required values](/consul/docs/k8s/connect/cluster-peering/tech-specs#helm-requirements). After updating the Helm chart, you can use the `consul-k8s` CLI to apply `values.yaml` to each cluster. + +1. In `cluster-01`, run the following commands: + + ```shell-session + $ export HELM_RELEASE_NAME1=cluster-01 + ``` + + ```shell-session + $ helm install ${HELM_RELEASE_NAME1} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc1 --kube-context $CLUSTER1_CONTEXT + ``` + +1. In `cluster-02`, run the following commands: + + ```shell-session + $ export HELM_RELEASE_NAME2=cluster-02 + ``` + + ```shell-session + $ helm install ${HELM_RELEASE_NAME2} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc2 --kube-context $CLUSTER2_CONTEXT + ``` + +### Configure the mesh gateway mode for traffic between services + +In Kubernetes deployments, you can configure mesh gateways to use `local` mode so that a service dialing a service in a remote peer dials the remote mesh gateway instead of the local mesh gateway. To configure the mesh gateway mode so that this traffic always leaves through the local mesh gateway, you can use the `ProxyDefaults` CRD. + +1. In `cluster-01` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ProxyDefaults + metadata: + name: global + spec: + meshGateway: + mode: local + ``` + + + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply -f proxy-defaults.yaml + ``` + +1. In `cluster-02` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ProxyDefaults + metadata: + name: global + spec: + meshGateway: + mode: local + ``` + + + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply -f proxy-defaults.yaml + ``` + +## Create a peering token + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + +1. In `cluster-01`, create the `PeeringAcceptor` custom resource. To ensure cluster peering connections are secure, the `metadata.name` field cannot be duplicated. Refer to the peer by a specific name. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringAcceptor + metadata: + name: cluster-02 ## The name of the peer you want to connect to + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. Apply the `PeeringAcceptor` resource to the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply --filename acceptor.yaml + ``` + +1. Save your peering token so that you can export it to the other cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT get secret peering-token --output yaml > peering-token.yaml + ``` + +## Establish a connection between clusters + +Next, use the peering token to establish a secure connection between the clusters. + +1. Apply the peering token to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename peering-token.yaml + ``` + +1. In `cluster-02`, create the `PeeringDialer` custom resource. To ensure cluster peering connections are secure, the `metadata.name` field cannot be duplicated. Refer to the peer by a specific name. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringDialer + metadata: + name: cluster-01 ## The name of the peer you want to connect to + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. Apply the `PeeringDialer` resource to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename dialer.yaml + ``` + +## Export services between clusters + +After you establish a connection between the clusters, you need to create an `exported-services` CRD that defines the services that are available to another admin partition. + +While the CRD can target admin partitions either locally or remotely, clusters peering always exports services to remote admin partitions. Refer to [exported service consumers](/consul/docs/connect/config-entries/exported-services#consumers-1) for more information. + + +1. For the service in `cluster-02` that you want to export, add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods prior to deploying. The annotation allows the workload to join the mesh. It is highlighted in the following example: + + + + ```yaml + # Service to expose backend + apiVersion: v1 + kind: Service + metadata: + name: backend + spec: + selector: + app: backend + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: backend + --- + # Deployment for backend + apiVersion: apps/v1 + kind: Deployment + metadata: + name: backend + labels: + app: backend + spec: + replicas: 1 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: backend + containers: + - name: backend + image: nicholasjackson/fake-service:v0.22.4 + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "NAME" + value: "backend" + - name: "MESSAGE" + value: "Response from backend" + ``` + + + +1. Deploy the `backend` service to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename backend.yaml + ``` + +1. In `cluster-02`, create an `ExportedServices` custom resource. The name of the peer that consumes the service should be identical to the name set in the `PeeringDialer` CRD. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ExportedServices + metadata: + name: default ## The name of the partition containing the service + spec: + services: + - name: backend ## The name of the service you want to export + consumers: + - peer: cluster-01 ## The name of the peer that receives the service + ``` + + + +1. Apply the `ExportedServices` resource to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename exported-service.yaml + ``` + +## Authorize services for peers + +Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. + +1. Create service intentions for the second cluster. The name of the peer should match the name set in the `PeeringDialer` CRD. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceIntentions + metadata: + name: backend-deny + spec: + destination: + name: backend + sources: + - name: "*" + action: deny + - name: frontend + action: allow + peer: cluster-01 ## The peer of the source service + ``` + + + +1. Apply the intentions to the second cluster. + + + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yaml + ``` + + + +1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. + + + + ```yaml + # Service to expose frontend + apiVersion: v1 + kind: Service + metadata: + name: frontend + spec: + selector: + app: frontend + ports: + - name: http + protocol: TCP + port: 9090 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: frontend + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: frontend + labels: + app: frontend + spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: frontend + containers: + - name: frontend + image: nicholasjackson/fake-service:v0.22.4 + securityContext: + capabilities: + add: ["NET_ADMIN"] + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "UPSTREAM_URIS" + value: "http://backend.virtual.cluster-02.consul" + - name: "NAME" + value: "frontend" + - name: "MESSAGE" + value: "Hello World" + - name: "HTTP_CLIENT_KEEP_ALIVES" + value: "false" + ``` + + + +1. Apply the service file to the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply --filename frontend.yaml + ``` + +1. Run the following command in `frontend` and then check the output to confirm that you peered your clusters successfully. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090 + ``` + + + + ```json + { + "name": "frontend", + "uri": "/", + "type": "HTTP", + "ip_addresses": [ + "10.16.2.11" + ], + "start_time": "2022-08-26T23:40:01.167199", + "end_time": "2022-08-26T23:40:01.226951", + "duration": "59.752279ms", + "body": "Hello World", + "upstream_calls": { + "http://backend.virtual.cluster-02.consul": { + "name": "backend", + "uri": "http://backend.virtual.cluster-02.consul", + "type": "HTTP", + "ip_addresses": [ + "10.32.2.10" + ], + "start_time": "2022-08-26T23:40:01.223503", + "end_time": "2022-08-26T23:40:01.224653", + "duration": "1.149666ms", + "headers": { + "Content-Length": "266", + "Content-Type": "text/plain; charset=utf-8", + "Date": "Fri, 26 Aug 2022 23:40:01 GMT" + }, + "body": "Response from backend", + "code": 200 + } + }, + "code": 200 + } + ``` + + + +### Authorize service reads with ACLs + +If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. + +Read access to all imported services is granted using either of the following rules associated with an ACL token: + +- `service:write` permissions for any service in the sidecar's partition. +- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. + +For Consul Enterprise, the permissions apply to all imported services in the service's partition. These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). + +Refer to [Reading servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` configuration entry documentation for example rules. + +For additional information about how to configure and use ACLs, refer to [ACLs system overview](/consul/docs/security/acl). \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx new file mode 100644 index 000000000000..956298fe3cd9 --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx @@ -0,0 +1,75 @@ +--- +layout: docs +page_title: Manage L7 Traffic With Cluster Peering on Kubernetes +description: >- + Combine service resolver configurations with splitter and router configurations to manage L7 traffic in Consul on Kubernetes deployments with cluster peering connections. Learn how to define dynamic traffic rules to target peers for redirects in k8s. +--- + +# Manage L7 traffic with cluster peering on Kubernetes + +This usage topic describes how to configure the `service-resolver` custom resource definition (CRD) to set up and manage L7 traffic between services that have an existing cluster peering connection in Consul on Kubernetes deployments. + +For general guidance for managing L7 traffic with cluster peering, refer to [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management). + +## Service resolvers for redirects and failover + +When you use cluster peering to connect datacenters through their admin partitions, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically forward traffic to services hosted on peer clusters. + +However, the `service-splitter` and `service-router` CRDs do not natively support directly targeting a service instance hosted on a peer. Before you can split or route traffic to a service on a peer, you must define the service hosted on the peer as an upstream service by configuring a failover in a `service-resolver` CRD. Then, you can set up a redirect in a second service resolver to interact with the peer service by name. + +For more information about formatting, updating, and managing configuration entries in Consul, refer to [How to use configuration entries](/consul/docs/agent/config-entries). + +## Configure dynamic traffic between peers + +To configure L7 traffic management behavior in deployments with cluster peering connections, complete the following steps in order: + +1. Define the peer cluster as a failover target in the service resolver configuration. + + The following example updates the [`service-resolver` CRD](/consul/docs/connect/config-entries/) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend + spec: + connectTimeout: 15s + failover: + '*': + targets: + - peer: 'cluster-02' + service: 'frontend' + namespace: 'default' + ``` + +1. Define the desired behavior in `service-splitter` or `service-router` CRD. + + The following example splits traffic evenly between `frontend` and `frontend-peer`: + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceSplitter + metadata: + name: frontend + spec: + splits: + - weight: 50 + ## defaults to service with same name as configuration entry ("frontend") + - weight: 50 + service: frontend-peer + ``` + +1. Create a second `service-resolver` configuration entry on the local cluster that resolves the name of the peer service you used when splitting or routing the traffic. + + The following example uses the name `frontend-peer` to define a redirect targeting the `frontend` service on the peer `cluster-02`: + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend-peer + spec: + redirect: + peer: 'cluster-02' + service: 'frontend' + ``` \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx new file mode 100644 index 000000000000..bc622fe15d38 --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx @@ -0,0 +1,121 @@ +--- +layout: docs +page_title: Manage Cluster Peering Connections on Kubernetes +description: >- + Learn how to list, read, and delete cluster peering connections using Consul on Kubernetes. You can also reset cluster peering connections on k8s deployments. +--- + +# Manage cluster peering connections on Kubernetes + +This usage topic describes how to manage cluster peering connections on Kubernetes deployments. + +After you establish a cluster peering connection, you can get a list of all active peering connections, read a specific peering connection's information, and delete peering connections. + +For general guidance for managing cluster peering connections, refer to [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management). + +## Reset a peering connection + +To reset the cluster peering connection, you need to generate a new peering token from the cluster where you created the `PeeringAcceptor` CRD. The only way to create or set a new peering token is to manually adjust the value of the annotation `consul.hashicorp.com/peering-version`. Creating a new token causes the previous token to expire. + +1. In the `PeeringAcceptor` CRD, add the annotation `consul.hashicorp.com/peering-version`. If the annotation already exists, update its value to a higher version. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringAcceptor + metadata: + name: cluster-02 + annotations: + consul.hashicorp.com/peering-version: "1" ## The peering version you want to set, must be in quotes + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. After updating `PeeringAcceptor`, repeat all of the steps to [establish a new peering connection](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). + +## List all peering connections + +In Consul on Kubernetes deployments, you can list all active peering connections in a cluster using the Consul CLI. + +1. If necessary, [configure your CLI to interact with the Consul cluster](/consul/tutorials/get-started-kubernetes/kubernetes-gs-deploy#configure-your-cli-to-interact-with-consul-cluster). + +1. Run the [`consul peering list` CLI command](/consul/commands/peering/list). + + ```shell-session + $ consul peering list + Name State Imported Svcs Exported Svcs Meta + cluster-02 ACTIVE 0 2 env=production + cluster-03 PENDING 0 0 + ``` + +## Read a peering connection + +In Consul on Kubernetes deployments, you can get information about individual peering connections between clusters using the Consul CLI. + +1. If necessary, [configure your CLI to interact with the Consul cluster](/consul/tutorials/get-started-kubernetes/kubernetes-gs-deploy#configure-your-cli-to-interact-with-consul-cluster). + +1. Run the [`consul peering read` CLI command](/consul/commands/peering/read). + + ```shell-session + $ consul peering read -name cluster-02 + Name: cluster-02 + ID: 3b001063-8079-b1a6-764c-738af5a39a97 + State: ACTIVE + Meta: + env=production + + Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 + Peer Server Name: server.dc1.consul + Peer CA Pems: 0 + Peer Server Addresses: + 10.0.0.1:8300 + + Imported Services: 0 + Exported Services: 2 + + Create Index: 89 + Modify Index: 89 + ``` + +## Delete peering connections + +To end a peering connection in Kubernetes deployments, delete both the `PeeringAcceptor` and `PeeringDialer` resources. + +1. Delete the `PeeringDialer` resource from the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT delete --filename dialer.yaml + ``` + +1. Delete the `PeeringAcceptor` resource from the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT delete --filename acceptor.yaml + ```` + +To confirm that you deleted your peering connection in `cluster-01`, query the the `/health` HTTP endpoint: + +1. Exec into the server pod for the first cluster. + + ```shell-session + $ kubectl exec -it consul-server-0 --context $CLUSTER1_CONTEXT -- /bin/sh + ``` + +1. If you've enabled ACLs, export an ACL token to access the `/health` HTP endpoint for services. The bootstrap token may be used if an ACL token is not already provisioned. + + ```shell-session + $ export CONSUL_HTTP_TOKEN= + ``` + +1. Query the the `/health` HTTP endpoint. Peered services with deleted connections should no longe appear. + + ```shell-session + $ curl "localhost:8500/v1/health/connect/backend?peer=cluster-02" + ``` \ No newline at end of file diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index a15b5169ac29..d5daac6b29cb 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -505,10 +505,6 @@ { "title": "Enabling Peering Control Plane Traffic", "path": "connect/gateways/mesh-gateway/peering-via-mesh-gateways" - }, - { - "title": "Enabling Service-to-service Traffic Across Peered Clusters", - "path": "connect/gateways/mesh-gateway/service-to-service-traffic-peers" } ] }, @@ -526,16 +522,29 @@ "title": "Cluster Peering", "routes": [ { - "title": "What is Cluster Peering?", + "title": "Overview", "path": "connect/cluster-peering" }, { - "title": "Create and Manage Peering Connections", - "path": "connect/cluster-peering/create-manage-peering" + "title": "Technical Specifications", + "path": "connect/cluster-peering/tech-specs" }, { - "title": "Cluster Peering on Kubernetes", - "path": "connect/cluster-peering/k8s" + "title": "Usage", + "routes": [ + { + "title": "Establish Cluster Peering Connections", + "path": "connect/cluster-peering/usage/establish-cluster-peering" + }, + { + "title": "Manage Cluster Peering Connections", + "path": "connect/cluster-peering/usage/manage-connections" + }, + { + "title": "Manage L7 Traffic With Cluster Peering", + "path": "connect/cluster-peering/usage/peering-traffic-management" + } + ] } ] }, @@ -980,6 +989,36 @@ "title": "Admin Partitions", "href": "/docs/enterprise/admin-partitions" }, + { + "title": "Cluster Peering", + "routes": [ + { + "title": "Overview", + "href": "/docs/connect/cluster-peering" + }, + { + "title": "Technical Specifications", + "path": "k8s/connect/cluster-peering/tech-specs" + }, + { + "title": "Usage", + "routes": [ + { + "title": "Establish Cluster Peering Connections", + "path": "k8s/connect/cluster-peering/usage/establish-peering" + }, + { + "title": "Manage Cluster Peering Connections", + "path": "k8s/connect/cluster-peering/usage/manage-peering" + }, + { + "title": "Manage L7 Traffic With Cluster Peering", + "path": "k8s/connect/cluster-peering/usage/l7-traffic" + } + ] + } + ] + }, { "title": "Transparent Proxy", "href": "/docs/connect/transparent-proxy" diff --git a/website/public/img/cluster-peering-diagram.png b/website/public/img/cluster-peering-diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..d00aa3eb0f3bbad88b0e13d6d905fbc346f2cab7 GIT binary patch literal 129219 zcmeFZXIN9+);3BI{&_p>&nU^S#xF1F~=NZ-1iuBg=?rOQjjr|;o;#?C_Q_k ziHAobi-$+}<_Zb!lXF#c7VZt-T~qNfUTOc$4ctFIR{BcTs;YQAxbrJ`MEJ~j#Fv-g zUO;@7f1NAf-^U~T{W$?1UYH#o(O=i7;odL*KH^@NZT`FyX5#;KHSV)ag1@gOkyI#;a!>dFF$fcTO6N&LuV)6P4+)_$fP%snFAVtsS7Ky(fIXhdDa zY5I}FY<)#Z-JDEa zU6(;1SC-;*B^IOV%q{xk*9uoA<1cTxXA*i%23x_cm3J3D6fL=9J7MCT?o`z$qO>M7 z=(&|=bWf!0_Q2t2p%$W|i>(UKde#O+uHQ)og8V+N?ZBI6ZrG%mr%QTAuSCTb z-NCI<+5Q#wzzg%Po9>Y<=c}nVR`kupVq`G&V})8=*(=bI$Mo0nl#XTtv1mlUUAQ%E94r8~Ko0p#do& z^t)sua{`^jZ(e9Hw4SfB z%=quNIL$Ttu6(>BVql2hqBrfgRT4Vxpjl!xv}W+u-+S6FKmHrlidIR*tk;awNm(G8 zSEn}F^eaKKcYwxMQiC4`dPblwR`0aFG;tv>rWU{BhY5pv!a}*~!1E60`b83>@|SY; zv!GKrqqdyn`Ep#l-L+*?*&z>51Mdd8mJT`W;y}fy@|BXK4LPGgb#}BwPb(JV?=YOB z^8dq_(%*9()6paDqkVchftymvy|09!rvgsX)fFx< zw8;Hj8S}O`u7WM0;+V4~#FPz$B46;MKfbVcE`wTtrn_ObLR6)=`|6p}w|qaN`mpWkBus_k%k>*Vq4H%}B+@@}R zc{(x%hS$Gdy0{Nf6ti%M{l?~z=Bc6fivfjLPwF4f zWe4;y1}dj@9_-1!iNmZfmG+GlB?5j&nr!H3grqO!6H^K?!Dt`Zsxz zQZoA;v|dL;iecxPu-5i>6jcgQ%)}!M{DyQ#Ml^jWdOc%vJu{GJy_zq@&kSkF`_D{F zX9iw1`6V6%Hu%J(cLRAXlS4}Ivx=OMTJdV4tLKT+H4b{Kc<*n=`WdjP7EydUtAr2e zaVY>=jx-026CruiBuJI<*ckAGBl#=<0F6GGagoc1Z`MC!_1;psWxy=vnu)$R99ZP5 ziOmoDVw2(93B1^AU+8s0)J%xfpz`md8H^mS3$UMP7tB& zE$vh?vyC1bG$!|$T93z#EX}CNWbbkkgG@okW3H9aO0jDnlqD!ZLxCq#HnnRR{?nxF z0n|IlL?h=43>uXReiPq=k_wZIzE+Soh}wm*Cu?Ft8x`ns?)>qzA60_=jyH*^9u%1r zYTDtVxgVyor)wB-6$qK%$KOl1s4OUiot_>~d@ar0;u}?q(2J3$6wHtAdl8Iw*CLY# zn;2SD4mP4h=dKXk0ZSnZIlC33Z{BFeGD~_n?x`sNj*k0_T;^L7^?&GZ@nq6aR2I-P zv;efKj6zANC{N0~CH&TMQYeewX8Di2QhKYdBmcAy;?U#=$o3Ai7cc>7**1myJ`2F+ zVZgMg#IfBP!=U}brx6Ttzzd(J0h`hpbAX}|p6v4Q-Y(=W?-;k_v49nxd713XoA77%hwikZb}VZ2R|g<2sGTO4VW41j zDR{ZGX-7g5Q9TmU-<~Z)Kt{H)_%r_%KDADGBuPxa*is}JMbKIEMuhwe%LKzcC0{)R zH9H>nz~T03G5~UZuq$LH^@H>>jFjCVrqhU-BP}3&ir!e%qIytbR3|V#`fB5}jroc; z0AJc@;|6cV6ucm0r^|gmk`~U{{@azD^J}~q*yG_l$2p&l2S=q`27B#*92D?)B9n3u zq&&X)rnbcUF`#)b#EkDu+l(HQ>%BYIg1|YBYa!Q%-j0SmJA!|*Cs}XR!jD^zaA;ZA zlU?2KVy7a&510F(0%&>U17m>KzSHs13r@xvdr{k8DqUH4@6^M0=WF3!GZy&NEuxXD zuR|$B3ij~cA3*)zs0cK4kX&PWX-_c6MNAujmm@V$HE;H7OoHpvt%3K56%BDLf?r2a8_U?k8$GLV{#|gKJ2r%Cx(+GIJ(>g1 zs{s^bMye0Fv#^d|7^mC7%3eV}^i#0|r9`%xc3Z3B)Fik?_>=DV*^W~QUi}@@23L}; zV;%z>j~6ynpwMJC9dkWy*)bvmr<$>0;G-mAs^S<~9qPf*QUdls^s!#?Xvt3BaawMm zN~^&e+u)oc*X^5xzx4of<%yXluby9B{xs#id%~f)Fl{CLYxH#ElHWa)h_#OA6oj7~ zIrHfFtt9A$v^5STbq-nSG|9j9-ECOQa6Mr4p1n>EUVmGcg4E2&QvJlMk=e?1WKggx zGNzq8#|sCe5R3b~gUedJxM87RLHu;}afT?=z>hJp_}rMDCWfAEtMQSx_ zuq87CdQlZ$sxl9&PFWv@gt%=6=2wOa+fn7bNfuW53}LMDlrIf4cxsSv^w^j&T;lO? zdx`T;dOYq?zx(dg8;b2*=pWD1C4_B(4%5|kCu4eM-&;TK zG#~aauxFXIQp@O+JWP3o(DF^Qfg!XZNYU(1ghMiDytqF>%zYw!xJ|(ZVN0e?IR9AD zC}9+MKHg)bl)1dooapnaP%J}KmgK!3MalUTGm&G2xrNwMYB z@32oaloL*y8WXtwCCma1DVp+z9QhZu^wV z?Af+ch+a!^&_uoUW75=w&>ITutNdJhiXZ7c4-(xze6^PDO$mOG#Y*z@PAl@;dyrM!H6e!UM56Nh|hV${^nyWu6NtP%=O>u*y$wBei3= zC7+C`;A5dZmm(@oWCj)_=F%C-AJjCRz{Ab_ zQHZhXeVwF*FoC<+l?_0Vv9_=#MU-vhUK@G)LKvhJz$*8Li-Qq*Aw>in^k$U)>KA)xV@ zfQ|u%4MmnAbXi4o}6+iafR7$OR4h&KtXQ_V%?>(C~y1hvGZjHl4Y32-b?`qaF z$4}2OuMVV#n7DB-ou#7glFgz^I`R1@BMdQdkH_Lt#3 z(;k!a{80M7JcRWUIPTqxKU-lkDhyJfnGug{TtiARQj;4z{Y_uRk7*F_j9-TU`>9w=RodxB3L#P0TwPzBlhr-4qUPUf)~(PF>dBrPrtq!#Hq ztMS!-T!1-9lf40+dVK!;8b_g%6~qmcf%ZCo@<(VA15gk4eZjB%UMs% zJ-t>Y$@OHhlmR7&;Zp(ij#Xajj3VD*jYBJNGMpUG916#veU1)4@RnWyyOgz@ z7#h{PntSMKe8S*>stB5!U3AtpG~ILDpu&pG>23ui#x}4Mr+z9++0EPwbo@6?O5TlFve`Y0;Xi~XU2E=M(^;#_ugN> z6Vj@ag~JA0+7XXV=K}*Pyz@3{bxhVa49BYy02+Dzy^ocA8S#cxeCMIf)7^&!ZwXR* z1B#LhI-X7Az+ajdV|y4?U5D)!4&*ZyXM?q~SPP*_CE8~3t^AK+Rkj`D>b>9d53>7n zkJ|ITCP|R$t!c0I2WSI-vmYiZ&y@@b^XXD}{$W9X64hzrMb{eMulK=LdmPX;QNqar9FA zeCex(amk!6_CP5S6|~)1{N$|9FMk18n!O^DfFNZ&TaikJHy^@tr5Gc7*rh3rNdOPwmMfyGNn65q-^ zm9y_rh5!r8BH`ZiaO=L56|T1svQ4Nd_gN zPbxy6=d3rj-p}T)$R5VgoIGhIhIfX3;6S693-Z>XlfdTZY#E-#4=ax{r@4nHH|g=m zs1kAPi?B?!zfPITHlE6eJ1C<@mF91CrqhVBfEyKEZcX7Z2)pl)jrImoDNW7v?WacvxI))MyMrY_V>7s240(l{n z(nlIi+m*c`{xYgseGzjl{-pRnnym;LBNPtgp zL)yxs=h#F&ho#bOuBM!XM$QdZjvyzQIGIy<<7IG~=FaVjHSv{b&7Vx-Zm+h*j6J{7 zsXV;#2l%zoy1sP`K|O~%G?MnUN#eOh$|e$&s<$jmdlP$A&JsZX5H%Wt1S5H>yl#LW zmb;t}edm9rwDP$1>C^Y-_;;H)8kOR?4heSMq}^8B&CZ&Et)@6w%(x1RSsa=~#ZLyv zWO=I%A?m>oE{bpKxr&Pn5Q}|?FT<^erga>o6>$3&IT^rMaCO2Ht<$k;$LS^7atx@$6luW3VcJSb>mZOU!{%je0-Mc*&$xse6*m6>8?{wER@>hXA)F#Si>-Ov~`T!>Wh?>MngL<=ZQMp`zxv)#Z=W3^8lSTVqM5?du z^$&cnFJj#!Hv%Qzm1IE3I}sOgm#!zl@_4bhq(gYkoq`Qz`=u8%s@qZyE3+R^FyrG; zoOiC9!r2iAd#&NIU)+L(4N?R5b|Q)YpyUFa7hrSW|Mh)r`-k}KCJ(ifZtd%(&as5x zC56i{?>@&R)tHL~Oe2?AFqtdiHvY?0kfSk1naTKC{AbsO_+S zwKKet7^OL4)M|IU{?Lk5*NmN~92(Mt1Fp{-#anurAG{2b#>FS!3B>Ka;A0QC8K^ZE zf7w7liRrK&*5vOx+9n#U zOT&uNx2*kic35fBr5Jn8J#X@=)w9J_Z#YRFx9WtI2p6S@A7Z7Pv|N$COW9soy-2W2tF0`;getYi(2yo}@Qzrj?{ zlEbLw099XFycY>~C#>}sa#G#lIe!r$zcbQ562AUS778U`6IZBjh ze>4bpXOpkVECU|H&kgseJW@E$oUP3ZLRS*saHL@b-5+^7nx* zD=3gjr%>u|A9;D_2D&|b>f`d*fww=e1T7yFSMmR=dzS{7gNo>gqtW8~2M(@LyzWri z#|(e}Trdyk0wIS4AFga*d)ZCwk9GLd*GoI`kF$^S1h}@lhw?jrxBYF}|NX#0jw?Vg z$gCjyx2FGor{y|aeZd7#W|{Ke+x|vYe;+ETIq*IaGeiH;fq%aXjtvHrQ~%!%{r@1L z;Aw;6cAjpc4DfCZxzs%#`m7l%C=VJEi$-sR_sUm;Hy!i;0nOtAfJjb*YBqHf~H#a^n=7f@YcaW_g@ ze4D@>biGu482NHf)#mj#J_sO9h^lkKwa5^ z9^(n*1cDo{HZQ*ODs*gP z_2u>ZoYY52XWr~`->o0w|7+<4h1_28r@$?QwmK531A~HpEj?{Td|`k2$9Tb;yyYRg zL7V;u;%Vjy3r@Y40Z*Y@2rXCcC8rw~C@A%7# zd`ytlSjY&uV=$FaO@sgO)0@vRe^2P2n``*tPk7({ha`pY$)c@RwWo2NjMt4^`OQQ9 zkMU8a%cQmpf5P`KyYPglK%D*Ck0}oPk7+L|Z~dDE{M)YlY5s!|XQs~GIRB)T{=FCf zy4~{gAiQV&_ox0_1=DhbEFz%$%K86PG5s)pnGg4>fnhFqjTf^Q+4ZyEMYX;D_u``9 zO+MGD1ieY#vQ`F~S)m1its2z<*RaYSm0D=BwFOvXPH2-*iYHx>zP-JH{DCcHUIPYi=uFA)!w~GT;so76N9Jv zQU~JR7yH|N#bVpajP|5g=(NpfoH6R90bG@ACBakgz1wH9wgZ&~X>_PBS9 zbnzicdi!gNpK0Q~rXaL1%Kk`CE{=KPfEb-j4%g0uB^_o9UOW0mT?;tBmn!z! zLJa+LM6FK3bMy1b>7HfG)MuB@$=6+MmNw~Nh5F}8gyf8T)f00!Ey}2T;hbnB6bkto5=6%?dhKjg51QR=w(`@oqJ(jfH^i?*K8Ax@JzoGo2;gU%@DYwSJcA zB`ms373<(v4o+|MMEulJS7|IU_NAkzpQ;*=abX7^#eyU@wH$U!Q=>$ue^5?(%WUDG zX#FmG7xq=CneRO&NZm_0F%v>>3Z|FGv*!gm(8P35XYyk6Q*y>>7WsLvEsEN3*oB1_ z7z3|H?XQdd$p z-RbAl66J}QzGqkJ(sW39*!R8Gq@{!vy#A%uXWznqwP9`hZIWrV66|?pocgv4IUH1_ z&f>XY-*mjOq}V>d9*7(wfg$b1TBq_-r~4(*cWzF|iVR$VL^E5p%iO%~Y=Z@zR7JB& zPM*pHxOmTdLqNYWjK&e8-5KCtu*HY>DCinE-4p|kzDRZ^Js-G^tAR2qSKb;gQSGX0 z^2aGb7i*|ZqxQ_O&oCH3 z^OSH(YAf%+r&oHS`EaM{q_tNo^c{s$Rg`E2PQWv*{x}7*Y3$!=#kRh@zy+6A^Boq> z8A+!_OS#Q++bP9I$Mv(F9`He9WLWJ0*KXQc!q{!%;6hr6lp4npTv^zRb!wv(YU&*e zC#*GuVh`5LP_W2WA8~Li45|1CR|mtvn?5VEjspQJ^0l8<&A+#OSIQT9@}1+3q( zhiJJOQMR!CX!(36*3oIEwE5O($Cuq}RoO8z{g}@ZV`Vg`+4^3ElMhV|xcqO8<;bb{ z%Rv7zVxEAM{jNt^HXf%JAg!Veo2G>>e$lyQMkQ1s3!()s!g!Ymk48^o=dstB)p7@3#CjO0 zjsn>N*Ucr1=f7|#ac@O3dv57R877$R`}C)Yi%b#s#y5H_;^!5wB^)0b$;S9zGUyMd z`lXvi?S^yOgqkgeXsRxJW{dYMZNh$gq}Rcfb4mSPr>_}JJz}MgChzQ1V5Rfcl2V<| z>eAg;4Zg2V*v&q;7UWoUJT9{SmeC19cp4)eXyQMnMBASEZqO&q zz+6m^XCLlOtyuu(t3V3%=u$~UbD$~H&Kmz|p8T%C-OtrePx-fV*D~sA#*Kw)-)A@h8jon2`c`-i{LWfQ9n;9^ zW<6#~7Pp78fmCF#-_p*?)($j=v$-@z|5R+?7_!ncqlX8G1^kRvoCXo@TqT?4#vAi1 z(HW>%e0-HKTOojsP9zofvK8UICnvIjYeziE14%!__yIl=4b_jw!t8*9Wrk`~E3`<>(^F2BLnRNG^ZX-^)HEVbMQzhKGJS z1A7bE4AdfwH<;n?6-Ky9zPuaFcya#YKawXaQE47o>ye!gw z)e2g9{<}@03>VKDUVgf<8@LyYsMVX--;)<+^K%?(m$*x0%J#ZO#kuu3aZ*>Cxxi)@ z@rgvHI@1|kH{;8N@*VD&^b5b!a5U+U1;ydMP*2d7T-vVMX4G@fuD0u+Z5a2@XSMtm zQ(LM#JZ|JJ;m|F|uI*`#4K`f*F(kU)VM1(jZ0C^EHfY{59Im%vK3(rrBe|Yxg zZL~~7poTxE@vUxI0f;96w%LOd7Xm4$7iuPaQKh7K6Xd~n_QmENanE#p%EO@?f`Fuk zfRAMAxDaPCUC8ZsD)gfwfqh7~UcPQE8))GcY6n{8G#bV}m1Z}Bcw-q&?k?$#F=jd1 z>{*R?^rG)Uy3YZwp087f28)DZtss=x9}3N=jC62M;bMe6VyQz*T?BrSuY4-y8pxD6 zSlj5aj2HD5&zxBnO5{K6@M(^OWw2~ znONZ)gQ9qc7A8n~F#THqVT=BZJ`vLl13TS`- z$4g|LS;;loC^yx3RrH}c?1m&}(lXRgn!Od(+9ZOl4BKgvNy{hw7RCm8bkpV^;b*Y% z)%fF9Z2)X69;(iR!1lAaOie&Q<`bsA2{qortf0nuVd7%zRL$H0Aa>RiY^5RAlny=5 zTn%O!g}NxRUZ)~z#;m3;x-m0JdKqsU{8}KSykfuQU|I^fUfPOjeDE<_HX_B((4nNJ z8cH(~5o%UDZ%nHOL@cDdN21r1M5b9$dfC9LG9a37{GdOq)R%WGWp9Ibg71J9i-NFI z0I|Mv=il6mGQ^$F2cq%}*JRYwn2b0jK5f~UNDCD{z&Reu@;Wni3#wW7vciP7wwid# zN&`xmWIIT!zanD=>23=kZDK6_wBc~I(~x(t=7H5PK;&2Y>2u*@*iCUBFK0K+lT$+E(adjs9185 z;+YDq2QZZ+3MYV`(DKVVSTF#_^ju5JlpTB#d+dg2=Tzy*M~e7ve+x5qwMIYi`Q$jY z{%CBD>13z8TnS1d;PX1T>3YnGq}vZU(y7;6=}dl5-H4O?fGvgNqBlGv_J-T+k~?%e z)-$FvB*NY#7ofwN{qL_WPs@u=`2}gg?ko2U9IW)JQb_|rhY0U)nbDqT&k6E(TlJP7 z^3=AxDmkO{gkK*PI&g`lOgDnn0z7dciv6_U4*qVOxEOJZ9|2s`>(*8^pb0*afypjvO;{5XSCf zW8;lF(3i$ot$C&)&h*iw#Ri*n*8V{Fu^f{sDI!oqEM?1vXhn;7Ame;qt2W(VVd(IA zo*gdHwkQ-g-msk&5sKkPBR3s1gl`7KedTdIPKWl*L)Us%6I|Xu+RybI9X&`W{g|kp z>QrkmXtVe&EZ6z5=)^#ypYHh?da+_PQPOLjx;lR0a0PW|s46@YUbVJ+kt zIHOZBiiw_gaPo(6;7w58G_H1F+nRANwoskc82?*G>qE2@@K7t$8f+b5{8yfViNHZo znT5NH^kmBh`x@<3aN~Z4vUaOCzX zY3M+=7H3zd&luLNZP7)VHZde`?zHTZ#kXiWw;o` zcukgi|Fj;;dMe_A>1m~=J&d!Ln%=HTe4!c#Kiy&R@Q^|8wnf;~C*HU?YzAHh=M4Xx z-$)LlIy`0iapMmMnZW7VVIDRUxAVOMX0FXnlT=UHdYqqX0e^ow<;%F$w3bovRW6p- z>TrtD?9I99g7df^qFP>zjrD!)P!Z#qy?mWroR1O%e_7FxBKzwn9%HWcmR*!`B@wZK z6J%$YF!t>(2>6=)%WI(1bPoL`#vSj4qSZ36YwsCe$SRdP>zC76^Rf1~W{O?3s`J=0 zEssYc#dz|e@1V%c_kBNIpMXAJ>@A(19M{cH-_<)tDo(^+*py0~t_#lR%(e~C>I#E$ z-_~vq-Px$z+|y8XFYK=D&}L&xAK&;~&%zjF$S>lw8hN8)|G9z}+$xDfb~#mbM7U_! z9XS&EJZ1-85@V$BEwQ9-mM2cN^4dRG*<6PEy`T--i6e(ctOyC>GLt5sD65vym^`J@ z8`;``q5-kv(e z$x`wi0wfnSOlTRk-_9o_|uaZ4EUz_MaK)DxW^3|!hpkK{+^@8{X3lqPKly+is z%pj_1RxA6(O&Gr|hMAayOv9a_)z;PiWJZTunUl&_X~*4bw(7RVdfx?}nTxIQ>&?gq zDzntD=6HLdn~$b^^|q?E1iq7%zU3ACIDuo)4v z`Fg^ToZJqE?oq(m(b)$poG0_%Sbe-F!Z+yfUU1w^p`6Bkb*+}lxZdbe z>sya@B*_3_#G5el*b2c?JLoH=0Z?}chmg(BNPgT_mHmY?W=UKlBWmGe^`Q@F&8aRbRD&-GrR;cp#dfiSM^o;3Oy;qO zCh&CMLtRD3f~S$<-9I?VXZJoLKz2~w63cX_Q#go9j|}Eq1b;J$wgTm zeuRn@bE#4l^#naBc94l?XaDI37bcaO@QjzX-78v+INq*ZqLh3CP3#hC+bp$~ueA%> zayTe<9|$igwbX>Mvp5e!DOLV|vC`E>tX=$n=1TIW}bNJ%;+!vtGcsHw@&oyhjykJ>mA5SVX6v zpm1I-{xEt<>W_)gQIbs`MpU;)e}hPjXbI`;)+hDV&An~{oee22egPhDSBWe{ONKJv zMBrd*ajDUOXArtgXy8q+m5>J18VT|AJj*K@#ep)p^7HEgW`t&bLkxWS)H`bF?-k>a z8}IWncJj`{71<)xc#d65QocirZ7O49famuaF7(T^xcr^=cxz4beLQWCTR!pCILr%( z`0nwR5!ZNRfr@Q|V-&$O&tv#6PM`Vu+?`P`%`ZeYWBMggX_-Q2v5@HME)2@d|0ukx z-30x}{IQ1odm7bSSO2j!Q=H*G0<^K3>SzxEVGKS;u#~~(B)rv3Lhl$q=#y0fjKoM> zTxq&6(<3#~uvrFBF+0l7`3cSbGABKa7@HJ`v3ZkHQ|WK5$xF0m>I(j>cPuP6d2Cbx zbx0IX*xFETbtf+V^*wDlApsl9V;=%b4Roq`y1z=vY#o=Vmaj&YEmZ><9Os>7f_M zOl@64ol>37K-WaPPHxzcEH!S-c$c*}?MfpJ*HlF-ua>sx+#4f12OTUgj88jrb%;!^4nn|XXQ9+S7wZFujUH~Ym<#jaYLc>XbP zFRCu#^8RT=XsW;PQ0~zVYL|L*uB$Irf0Nttn? z>GRU58YS2TGv9cQ(jEWh;II&LX(Fc?hnWTSb}Ff6w?=EV=eCmK`oY+LM6((sm(i?q zvr_Z<>|!akUM1cS+|DPfT?#XUK@|h(lXc5s5#Q7uSDZ1Ne}I>@GCEs9a0|%l^A!Vr zaP~9b=8d3bYt%7LyZzZzuYvE6%wr!2cwHNcZEf1Mqmn*(CbrSmD^T%m0LFB(R6t>wQ^4awA1W6^um8ToNK%0mVBFC-JC~UjW;@@z^*u|(Tf5z?yjoJ z4*of`_;3je7NO#L@bQ#D_z)QOiuyOfA1p<$@zHX-es}x39Dq1b^kQxPJF-i^Ho&0x z-PPE#K$@9n&r)fywyAe8VALj+~TI?;-WF+;_JbUS3OJz>W$u^5e z5TfqO*9i4SapyUyx#a|9MU&V2_rS54hcK10`q=>4CqfJY`nHS>slP^M_s;^xp++Y_ zzP5CXboufwM>Rv-Tx_#gT50f>iCu;XYR9~8)(eV0{1j1w z<2Wqt{j?x&iVZjz+iXb}s3t-i2j^_zYFQbx{7F z^d2s8D?r@iW>!RC?6-5U(G~%ybm@%T&&F{H-qTJKdj_vQZY@19!PQqL#6=cpOm~`y zkgYv|B&R&Gw`i%G0jA{#2ly%IH+xup6NY@-5c-AYuy3??mS?P!`}H0T~V=2Kac8x!0LlzZ4!x>Y``TVbPfBTyYhD?%YU*>%&3o*r<8c`TJr%e zS7Jk~dap42Zte`}Ed8UWV^ubQiTrL0J=TjT{gMcT-!WOrG#NwxB35L^qiS-z^tI@` z=hk=)uKaX%6tgY4+hU1T8IGKMtVdzo%Y7_)_HqsFQ>?XTzFXfaI15qVTzZ8(vNJz? z1#ZOtWQQDMO}^viLGBJ1mVlXZLXyLs(Yl8*vv-}m<>*}RErDkYn}AIN@< z-I)vGm4c|6Fs*$#^q581UVBf9v9IYJBgQ1otlfTi3$m8(f^9wLj3U`+dcCyW#Nxb{ zohPeNx06abL&c>)!n0%`T8VPi0A)Oh%V0dt|N7$xmBOUgPO|a9hgSMvmD1+6PW7`X zFWf#42Ypzu!)>2egG9O>EG>0#0f`=0eB2WZxP#)JF#L`J4)hj=BLpf0=c8+q%nYXD z6H@d(i15<)H53ibA&}Vjd8rssHf{^=3`t94rIz$@*M84d*;B@=@8~aluq9u{B~-&e z-*uG*zsv`dGG1=e%KC^f#D4hAhIuWgrSM?Zj{@t{voyC<{c1B^zdPDl1r2hTQb z%k>5_%pp8Q8?HkuN#DJ4S@w|>o8Rq!{Y()T;=|9g^gSG#Ko~RLuiyPfYO!Vi1eDW! zw0&4RQreVrEoeES3+TT~OCEF|TZH)K(7b&RlaA=zaw#V_>0`KOVx(u{?erk+42Ij~ zH#t+*a-B$>W=$~qf4EOs1EL;3Grwi?u!f{znTF{ZQ4&=$s6uzsZcEZ56{-8(){iqxDFi-#*NVxL#w^ov6LJ`q;*4<3yzOv}_lAxhgCvQGys}>_ zc;uL3%BD;@l;eFy?&yVH_0{tAlbx0=XWRsY(EV_2X443MmE!oPBL!ZVf;`j@i2|YUuZM1kl5PL>@nXLP@ zUEGPNO>Yw98CeIH-?j&caWoRYJE(yxp;TAgamr z7my~7uhb5k(N;a|T5UQhT;g*9dtxgc7+g?YcelkkV0CQcymx@Y1A*Du=K2n#Ka)5q z$VM!kh8e}M8Fj2)LjAvMd1On8F0C(TUTNzu9@JUOV(qc65xsxn3pP&}>SRlPTWsP~ zQQ#Ajc{a=c0;4}5HIvVchENS$kvM@0z;649Xx+;1$eBi(fso0E%@0LXO0M7KG{A`a zegHl2(BGT=ViR+Hx79|bks!#V(a}~bUp1;^xWGJn(#J|_tMlQ4!l>a_D4@krx^9Z| zwXRqz+G}Qtt5oEDMnpg_{^CyEFxDkcD4Ftb2qO{Mv^a)0qO4P?)I zggITplT5<6WSK8_7X9IEgjT;L$>r!|j~PBMOT=Q@Jv-2AkP&#q5oayZ-l^EKOO4wl zho)wftDR~d`*H-E;*8tal;^oVcx!ys>DNJmFwk@Q(S#jJOpBchF>cr^Q9SbHabQzT zpyxFi|8d8O7bpp5?ADv~gIuBW#V&SZqq5%irxHy(Lw=bb@&xC>4yow*qA=DmGT$y@ zui@ODlh2}o8GRYjTvv#=UuHAQhby6bpJNbp3&>)vy67emy*ke{qiWTa4o^2>({@Ce zLP@=QX{|{L;^%8s&gU6Y;`+HbUz%%>V0Hbz7{2uChQDE9`qs}Xr5yCp2Z_&FPgU9s z?dWUNPeOnD9xQ=X3qZ&HsSNtj94Go=_jI+hC1R$oYRshMO@K-QI`V<{oH1&n3u3%j zDidPJ;wRuVbbD^i@?q8dEc&3i!M)AEVFQ@8Uj){FB+vS#&a|rJDT>Z`!B9lQyh^*U z65`3&kQZx%t8C9OPik7S+fGRrFfCWN#ywR4>Tiu~PvkTGm65C~#0mrQ-X1G3nvoX$ zj`AyhkG5`#Sw{toF)7o__IRNh3}ZOU-R^aJ{VBn4CGh)F40DV#RbYIq6adH59aVk# zu-EZ4@Y_?yDX6?Dm8kEQs_2YCemYyU!@}{8fT-Om8&Ob>>eMP1LP15a>UDN!9?Zk8 zM#SlP;6ESG!=sg1e46XmjoZ?mv{3diCOZeDcOn!L!f?5Z)%l*)BXbNnfbyC|{At|j zMpGR{z{!+;*Lsl^I|cY?%5$O`_xJ}WZp>rP4Q6Vc(B}8h@yRKrz?!A!B{cJy494pq))}`~ZX`v=C)xQB0)YY8KgN}pIatVFKmxn*1ZwSO=^NtR{e!#9)DSypEZioMl) zKp93X%w^q|VO7?$nDYOy_m*){DBmBjuAl-U3IZ395b0F9OOftoN$FU+I|LLYq?TSp zx@&1z5s>bVW$A{MjwPN|@BRJnFFyC>^X7TpE$n=DX3m^B=giFad=IJMV*3I~5Bt$n zFrE&|ZTE@+qJy13;qw*;Q^&4QLNlUe(0;lyXS`>0hWVV#EI($nUGYzpO{M_W(}^;O z?~g?X;tdZLK3mM!s;&>_b6st`PrCC)P9jy)clb2_b{p|1{{pbxkXt#H>~o{0goP6- ze9^v~3v16Np0Azrh&!GWaHS;su>B%ROcF=k&!msgw|xiiFR)8VfZY(~9}Y@GR=#;r zq^Q%ytej{gp>6jQ{Q*sdFLG4BUtXB+>rbpQ3`Z`=N)5Dn{*5O!W6`%|KSPW^8I*li3|>_@j`WZEA5+YJA$ z#UloFG&(B8Ypsj-Uryt{d(3`YJLYUBaFXh3f=f9xi z|GUS)c(f2zM_%0dhc(64K+paESD?Qw(Z35T^Z!?%|F;xKY*(V6@0UI^p_5_4WoKPm zcj}1tkAwKNOrQMws{CB8{4BI~7jAqg`Phc`lsKv9I}Xgc<<%T+hRUeo>HJb|(ab#V zX{eO=YGZ?i+j-hK&(*vZsFhw+!#x~78g~7C*B0u0TC2;N5?{xHvjYCQ0~W0&iH3m< zlY6S>s)fAn+0@vV3H1_Ai~px~1inUZ&=1rfEojRv=gN3|zs3|#lteL?B*CVdrTs@q zLGW>F9*{OU7WP4sh0q00YxIM`xvZD4>A7cLiqDM&&Fbz(tVY~wjgB3B_i@X& zqB3HT)5X=7^+a<}?Jfnxo3+q>({iPDga(sbKoECMTpp3zX48S$=5(;s znw;i=0Og^S4|wwYxwD2L+E$H=c0PUf@6JUmb{I}GKzl`d?3~&qD+ILTKzH0dg*Ft53 zRsx;T3hI9!%)i;}l_Io9@Db)*!TE3f{g=)D|M-B=DCqt_&LybG0z+?V?aFn7`vvYo zj(K(I=lZhj=|kH5g}Jz%n%@ePz<+PBJiHC;eAN7%IO=+u=p(72bl4k%$M-Q@BI!{N zC;K+I2Tu8j|Nc_>dqLxA(7ux9+1h$@fI%XZ&o+z-a@<&q?LlKMBv5u9ID4v zOeo=cdFy>b1kK3R-d6!OO+xC^<|0_}o!kTN9wO<+2fvQ^G0}kQQ!P*C{;GIC zlY~op%uc|=Yj7UG2JYq)Q#GQutgGp-x9sVq+O5g30PB=<_O&td2W`h0Nq2RNapR-I znn*=A4Zbl=`kkkKj^N4tGs?Tv%d50~ZlhQZk$I!W1{#kTkT!aVMW#*O!-7-GnM@|V z7k{$yC}KFBqlyr^H~5hjxoK$>Jh69fm+n8Ms{GJI)HTk%hwwZPUiF@n`=OJ7o}Bt> z6|f^pftFvEY;_9YR6SV>%M%|!NUJ{N_bJvPfvC*T|(Wd zIkONZ@Ip=X)~oa+#)3mm$R%zArP7HEW3DNaJTrUX$ov>hwECfQ%AdTlE1x!!D<~&cSEWqZvxXqM3>*+~=o>9JB~)!ZiGv%;TK{{#G#K6P*Y+ zG2AHdllO4^9-{RLP*B=t>M89WX>-2eTDq@3OLz8+TvZY)yWwHnt|0=pNx^c(U5a!B zX?4RK_Ggy`(Tn#BsHLor+4H3!K9=SDidE+QdHoXxxgTxBbiGHy!GpYm$_7D$;g1;l zIn}vDuP-L5(R$|-CPhe68vav8OFC#7#cvY($*_m0Ic{L4_%Q&R3CzUi7 zs2U!?U@Ewwy+E-^(_D79tKJ25Oxgsk@g!AN6MxZUmK&j0v}(q>o>-aD4r%}hg7~s(W=z(N^r=q&?ap`io-ZJ0W9XJx4hZX zdH#^SjI-PAPys!{mh{I51ot=OVA=7#?AABIF}9FlN-DVY@Jn?L+S~+9u}KXXS}@Zx z>H1w*wMFdeC|3NFu7OqVj4j%zAj%fTM*^pE?lr%MrQ8J=wn%j8R7pe_2y;pr;)BhA zn+IJ&7^+zDF7^e-Jx=;`2FpTZIRXUo?CgPPRZno=&11RI?Abn>W&^C!qRnxW>9pMh zAbGAyYl7HShP}8-#8R`^4L}Q@QeWYwDEW}Jg(xtf(Q1rqzYUfVL#%G9N*XMSP;P0b z|4#N2(|E$LIj`E8R)jEUreP~EcZ_IrdvUdAt%DD!fAsS!RDQnA94|Mx)T%`9Dd%jy z0f5jo5+!EHq)$k-EGU%la1`hF*{9w*4XFnoGnLIx^`>p6nZSLBg=QOzcQN^r@`mud zJbzm^5Kn)ZupUtv?*xKiF^|531x0zn!+lWI7<#3M9R;FU>+)RUI2QUlKEHo)=J; zITC6fii@O>K1R%wIe5t}Eyw;2DX180O>m*eeHbvgSi=beDH|DjU*g zW?~_JwVsq8D>_6KBlp-=9d>K_^}2Hmc>K?5OLa06uBgXAh!I^1SP^S7*SNXRg8GaPm9h+=_l zoHY%LbjqR>8qHz|>plOxK`d+#v&w)tS*hrt`d43;ms~#TqUq8a=BNFkGQ6bW@89xW zvhumc?aFXPy=6kyf$JV1S^}OpRadH5*%4kvqz1VK{nNfBn|8p^2;Lb#H5*r3%G9fo z`>B36z{Ml=JD@VYcGBZHh9u$RhCJK(AC40j+4{xPezTaP*4$Q>YW9C5S-YZduT;f+ zLD_Q2A}pnD*CG?9>#W}txOZw=xJC18`$-N8t~@)7e*WApQn1Z>qRgiM!VUyW^T}gP z=Ywm)c9e^@0ilsKd)Cb6?BzP_QSwlM(Z^l7RB<*UbWA7v{b3=Lw4QDskMO=!acb=B`Lhokro06THoAA$I2{lw|Toe(@Cb>=--w+2pA7`{*Z5-1<+(~mbNKy>+j&w$;7IQj;@)^s81;$&NA{>4lD zN1IYHJj?OXEgc@$H;--&AFVnQolO%S)O&Q}r*gY1+8Qd5@z&fu-s;RXRfgPC{Ul<-KgT)~Dt zafqibG4~Hg8$5A}(79lP=HDR~^Ju8Rbm8iH$6#wANiyDQR-)Q=oI5&K;BwyeArMjt z?!Q_90$tv@J7zE)k;e?u zXtlxgoGG-+9o0bf9_0fmpf)p-KysM(hubj)ytpnZXCkHipS$<&nY=C+mV`$Tu|rdE zx2BKo$(BmS@(}nBrU_d_vF0}l)mYZBALk9GJf2nDUSqAo4q!O@R9A8WntTru9DUrZ zry)k0;^gXsy;og10(-CNhyFagT-ddUU*>pfPo;m$wBxelq!X4=y>`#u{&4H3dWtZZrq3sK{1rI}`u_r%n{@V_)uTVQ5bORi%*C0A~{O5h75)~BSl|W!?WMc$58QLX8(_>Eh#a<&oZ`*mtD1Es8P`+QE^6Q}RHPG)3r-YG4GfBe%Em zMQ)WUC}Q(Z`*>QXM*#5Uc zt$D)%Pn&A)>&`Xt$brnS3J-%QBlN^eko2X+`b(TyLAujpUN^_lj2+n4u6sk>_BHB| z51y)im5gG!LApI!TpBsc5foavZM5S~JuA~WDucM;n4(!I`*Ht4{5w>Kq8B{H=|VSc z$e?ixVZk|4WSysSmEW2g@-?r{v$kb_REMj3Ez|wjUDsO6eQK0h53)k2)@r&(w*jD; zqfoeiJz0DfaiX|815BTBm2G59T{)=s*o<{(ZgAbw?u5>SEhj%@apm5p=PRU{M zLlShV=#p8q+?&Ti-Dg%EvM&xLEZ>R`v`)cLc6lP|FSWs0fHa^qUW4CQ#S6a;C=y7*qJ7b6i}FO+lxX(8 zsquMZj(mn++sg-_#WEcs@r)zqm1`s!-;UxJ=rce)o#m32VLJYX#-t3!VfiT zuU-UEe!hPvghlz{?l8`|Ww*CDj;U&)Ht(~D@7d8YgEwcu$#{+IeiRK;Nz{Dn3?5(q z1i2FwePyXp{u=J8J1d}+W<#D)Kcs=V6JyFs_e{A`j1DR^0+M8RS&x=R2#xz_S%1kM zJE;sV*=Ml~v76N4Hoc(>0+%qI4t2ujRHM^2j`+r39~O;Tkl(rEQX@TP<7>OCpT zSUkae@aSzxg~moj=*ek@U8HxJZR{&?kUnC&#VLe)W4@KW=T8Pa^a{%?CdWQ3Rcz#; zWnGfR1UL0%Rm7RJ;=xo>nh)@OG!w<(HLS98uDdm~yd_PRePzZ!o;Qf2BMID4=2w~K zaiLc3Mdn>k^5wumrw=}%kN~7zxo}4VIh2!t}xm!(84oPRPyIiK#jh z+2rKXLTj0V>Fe2$2rjs{++;H1k&WlhfTlMzf%i>G6k;6un|Ey|`}lhFuvhL*I0w8qXpT8P}AG3A-ml6b3I zVf3lrL%>%JguTW+<7Q?BL)*I5nWH(Hjl;H6R@E|RZyV34lhXe6T6|tZyJcK`1nG7e zx&1}{o5-EFMija&NnW*PI%n+v<|O_JJaq63tP{^|XK9ygp*_+tfb z8zLA{)}b11*{B=VkX04{Zo0ttP*5S3nE;{r2NBWfH6-`U15IzkYieS8Oo((CV;!Bh`aI zrkltP9lmQddekexyruaX9g(Gb2&LX?-_Z~p&HynYfc4$dY+b5|mQ>3~ty!Vf39UZ# zdpm26x%X-L8)4qcNDkV7TI#+#$WGdR^5jgDroI3zXluvrg2{9mfkt}*So4}q9 z4?h&q)_@Aa-DFd=V0OYq+Xh|h)Mle6AhwfU9$ESrv@L0VIhb4 zj^$k|wZ6P@W8Wt@R84*dtRp4I(T7a-eXRp}C)4LU&8PF; z_EX0M%tT`0@1-A$7X)RFWanox1dRZ@c#+?1-Pqm1jlKF#tD=KFEbo3C)?;Q`aHC*=u#-^)w8-eyg@uTf7m zcJS5Nfb&4GVD32PYscPY4eXkG`8%t_da#1JXhHfIUtU;Mp=~;8bI;OC z_ja!7)s?7cc(rExlxrdOXMmWQ$rBz#oTlK33Z0y$h@eE_Q_q+kw=0=}%DKC)E7A8v zusuJ%F3#7j5BD5n{o@YJ|b{iUff3#X|*R(hrQn@nQ*>AM2L0hW=$}6zuQc%#paT?SiY!dPhVRaF^z&DP!g$lfWL38S5jBmp2TB0RtuTRfH z3nJv3By$C8uwzXOvTSuOwc4Xn*KgS56g{Vi{o9T@Dp!uA`s_V)rY}86|@O zNe|q8;B`6&gPn106as_Qk1C9}!^vTz{M}EaSgx%Dd8A~7P7$?A!fb2Qi z-W?37i={p$5U}FSks(Vf@XWXtVQW24@Q5KQK%_m)oByemmOT%yKt?1e%Ni_svB`Q_ zK5Kii&nwtX55Ri-zWPm}@R?g&!4!VpN6C4_HqGE((w{upBYIS^GyAeXPK&;nl@wfR zKy93dErxv#V{O`2>JJ9#<7h?8GDAHXO8rHW7<6(qn`*AlP~+tK`PM}~APvLSe9g*> z<+AL$5X#R{{Re$S?~ks&ismj&&Qw(8%)DgH@fi&r@NgzG?*nr`r?xr1p2T@hYrlIp z52tt>so*lC1w%niwaQrPkGH!Fufi-&LKb!i=pJnf?_EYkXPEgprzT^^x|;Rc`*$~B zIlsn{3daT#T`|HC=c1#_G6bWT6MXaynhdDz^JitL}QtUA2Y$LIhmpvuPgiE7w%NRawfum?{V3Uk!@52KtovS?-03U);*&AJUM$H!u* zsg(5s9Zu`^8ncm(VzNPK)a=A+uYj zEo{}hg|#sJD=D!|K7cWEbn@w)Pj`18za5{QX@*`$h!m45pBWgOUv_P{w-epRxc{zK zP;!(Yd~F>{MU$8rG7-4UQc5DAe?)mZ(B;t!@*ILk(rmY{_VI|1#9L@yt+FYkU)yXq zdxWU(PG`~J^;dAH8-WR;9liq?3q}#gS$3-HrB&JUsgttjcTQLe+MLq3h{{8zKPy(y zARSrfX1D9*%%V{@HW&VOluR-#zqqKysAw^dRvq$<4AVk%2c0lf#WV9y@90*928c1? zyjD$0?^&0cc`SYRHnv#oB2&y=h?olc9-j)f9=Co`BeNnuZYOw8kn>)_%+=6qif~_x z_hKsSVF{mUa4wiA9m|;w3*gs#Tm<)@NM+{tS?BL(ggf^c&1wp{h`KxG@XR6D?h^bu z;8;CV9z)E=`s7w;YBX|l65^4r(Z&tR>9@~?S$1=fR@ebBQs5)9%tVpe)Bzo^*_F%}*x|S-OK-wr=?43y+26nQE-4UG4a3)06nYs!tfUuEb>E z%d1=C@w!^a2~LqGF*Cdw)YIHcocX?tBTkkWPtgld_c&Hn48_W_^-Rjft9^Sld5?aq z#U)^~!x%VAIu#CJh^SwXObjtzbU}pyToT z>=%M#f<;SDc^zg!{w0js)vQV`J5K?CO)B&%3Fvij35R7-tQ=QIpbq2>6F22J;YuC*Yn9o7_+@yhu%X2DkMXDO_SS){0#0;QRv zJBl)ukVs{eE~jy5!QSm9v${j0C{Q7~>1Z>W(>RY7)|E_JYR6qj`Ac6crUYXqYWY4-^;~ zTG`9Z7M-SFTv^2;_;6?T9?usS)*>8(ND^CuQKD`9oYEUV;?)e<2EVn_wWtMXDsVAcvV{3JL-1g%s z5~s%HciO$8@A>(=7{Ig^Tb{Q@QTA3k;abb&)F`~2jt9xgP}#4N>Y!GO>nQY*+*kVJ zh)IWMBi?%cZM}OsZRg}=eombdsBy$`|DoUGE3()w<EyqL}H=fIk7Sa1tg z1KUK%Cn2>hgzi;pv7^bApw*%31ux$i=jw#vP2#cPjcHUB+r%hVgZ??4w`_=c2+;d( zWZySc;whn1t!X|RnYoe2-JCh6o*WaGQS-}+n3XTib6F!a+mGj1RF7|V^2bcp&*l9B zU6)~CkpCqa_}v!LN##7(V{UmSHruJwe8Bs?(Rpsu__kSFJZ2@lIy+jcYZ@A^&R;0m zb^BlFJ~1Gw>Afk`DsUs(ExN!kf*aZ>q4ed_z3T48y}=EMfX+7?3C{m63q6TDn|PD zmGJn4+b0Zf|CvGUJ;(m)ty5W;A;#~A7!L8TH4WANqOa1cMA)|wG9?O>{Z*MQPbNe* zng>%)w1;=-)f=> zdO5^Y%NR3@bpyveTFbJgHeS?03XE)hSa>ReYzgWvXMtm&9eFKtO76M|CC^TqluTM~ z|NZ#(3ckAlujuVNo<{u*uX`|hZ_ylh>)l8 zEKco_Dkv+k)Pc-1X&Tvf8P~wEO?8&>92x%#%p>DhHr^hMuNY4hgLlME%Q_bq>Fz&_ zz-P+D6PVfRnoyP&^ksRl2Pnom;YirrGjJ1F4==JKexDyvc0>1 zKmJ|7+BiYF1-200-oDu5H3CK>oC|!<2gnxaBN~G{#onjAT!3zBF`O9Fdi?C?)VHsm zXWj%I>;EzQF#LTsp2`!SCxPu-?{GYjt=iTO?_@KHd@P-Hu|*`XDw?t}>N@3QQnmz?5x`cy6&t0+eP^Ki`q_Ie@%y4DBo3q5Q;b`}d$=9lO$T515=;ej(7na2 z%EtV9SqyX`9D+eFg0xWq>;qM%e3TiSZJXt6!vNilT>awj+s$I-M+c=;N!Y!zs$#%n z+AWH8-X@wdnocGKw9AuKgP0HlzY@K(bl%Ro>wZ8X@4w>u5w9Qp`rggl=2D3@1#Z76 zz!N5=?*2M+k0f$XxD@-HDdGE7ib)jk4r@y6abYam&wUkW`gmwv3|VDFGx+IKHeAHU z>y*6PtzDezG2o2e+@{6|)PJigN{e3z4!eC5DO#?Kv`iO8B0zoRq_gc7;}kx1%60ZX3kYeoQ2% z|8(or6JP&9Z0w`kd;9OoE7g8Y;sIef=Ccbc;4pJY3yYhGd&`R&_8`~f7M9Vk2-C2@ z8&$Ed^Xn<3eHs0~an%7*0f0w%cjNw{ zO$^1)^Gv7f3d^7U{wEOg=M&V|=VG_t-wT%cZ4LitLuov;YK$xkI1&9rXA6%rZ?y-v z()6Oguvhxl76?@drYng$0UT-pZ z>C;I7gu1)wB=jlf7Rj>??$dm6K58mpZg)eb8zQszi}s&JOB?ZVrmBDC zT2}_mRbBF;(s4j_n$Pzo&Exy(qWeGYzkN(QjZp}hk*v9Helrn$!#EwA$hBE8W-MCZ zckxLkjsib68NF_%Yp(0Xzr*Q&Z9!R?Oo<3zQ{<^#Jg8_QQ59QtyL`HF1ny+5UqN0_ z;yi0rkYqdZQEUakZ)RFsbRm1)0ep_2YkO?+#kYbp^LBEcFaE>+c_fSeS!IqtkBR?lr``&T zC(OJ>W5!O7_4_ROzZ)r|C;l$hnE!vO{O(})mFyRYg-}$STkzx#iMu;$F zM>Su5M3Fiy2y+v5INwNHYrUF(BYd*5j48;tHdeXX>)UDXXUUA9z_ax$x4#M}WcJxk z{x59&ZH%R#-l<7a&B5RBBgL*aBjKi8^%rkSvdJpkQ>UGt?b7`m#~tRhS-9=Y4%}G`1|Fu7)HP9nhJ?^39 zgVR4nXfdgG;d;m4IkCrg6K4LUxw2A5h|tIt|FUKkfqU!u|1m-IpJK6Sw-if5Og$Yt zd{e)kZ5~P5oyaosYZ%G^0J;uIoX<=1>b?wgic>1T96FbfjrE}&odTXMlW!D~Rf9=O zMw_X|cA7dk^g$xhVlXkess(J;3+`^c$DLq)kDMip(ygvqHy7-+t3@m zGp3WmYri+J-TtnOzAN5pbk3}G$w$zvQa+8VJ94zpOm5TVwUoO7y=K|9RA}TMb|XA$ zbV{jAtG!@w+YB^rBCI;`c@&=>lUCtQL>VV#s-0{ii_Q`oY-yx78(r9o!+cKLbGM(@ zY+#4qX}}-VodDL~`)>Tp9a6bsoPfrp(J8I^ra|r{I#peKuYYYh2G77YL6?||<1IqZ zh2moBFFz$7tBkf@G))Wz8QFyX*Nl*dWOUAcNzXaW{pe?F>=KCjfzs$g7pqBj!r9`% z7g)j_?441c&}G}ZzUuNAc@c_MC;aVEHUUks_!b$7Zkd zkvflk{BzSTDtDc7J{T*Ij?#y3C{oR4QrBsM#G>v{;PV|D+?yTF19%X$w7=2i}?E zlc$OP8U!KkAx4)+Iz@z8m?dQr>LjA9kKc3mX zW8p?W$E11}zYPgjwW-w|nV92*H@bq|Kr>#vj9kKQmwd^5j#)n3R#~=lbt+LC4oNP^ z8HFgm%O4VsuUOx&hOnQ482VQXTC7ItJXb2DR_DVg;&;s*H0=N2bSvicM(MkE*|5W? zrCj+(8U@%!Is1@H{KN)SRs9G&`=N99d4GhC%kVjI*Bvx@+jyad^A{GKCT+WxWjFWh zCcpJAIrgRnv=Tj65_{SYJFaY+q&x#7U#|Wg3_E>@IAjB#qIWumxevl z>lx4ei?tVlB5reh7A;nBcg!|2#Z}p<4v!`RZgcecM1%Oy?DDHMWTjwY1FvDWSVO<_ zj&$?idIQDqqV!G^y8IekkYK6$t8%~swx0Vqbfjqg<(?4w0y4(nXIwgY%yL{sayA_1 zw3q{a|Bz`Fw`O7&y4utI1y7g-oaX;IzWapuwa0QN#}&FlS{LD@-m`NOz{?)=f5r3hZk7KAeK7Vo&qOPJ^QO&waq>o`-#TQwC3CLM zxY-`k7vB=_FzRGZ?rJqO)jqZ)_t8*jGGZ`Z9P-f3T#0|v)U)J9PWAI!%*L6kv7m@$ z#SLaM9$N>nCw;Z@>ij**c;}1bI17kJPeI6@epzP|-2!G&Ddec1_aMfz_3ErG+T%-D zTGa7REs!3`*8dC((f6jWJR|A_ zW)zPlPUJv3j;_DZ>0Rwm@)G*_dMhr_E~+REIo=vBxy`^hPx-@EY6U+y;W zi1>ZZs)>+B7U-Bnd$UN-u2j!&pSFc&lV2DW)V;9tpgtG6Vn(cQYmq+aqc*%$u%2;z z1&1Adi^%rDym77&QB&k7;mku5_4Pq`m2|5u*+8?^aSuVW-M8;??eC5b2%Yzh?)hzx zl|Ijdcoo{tH|fk%@+pgu9(`cBGAj_WXA-u=KI^I0k^z{qc8LPRhMR2fJrjcBc#=VC^SsjiMfy%>>`D|J5$uA$F zO!F(j z=-V!~zJUQZUDcAyZ6%$h0>+aKMrF|^^+81#Wx^}nn%EH08y0)mO{tHqLH>rj|M=Fp)8@EV z&!oxs`u#ifdre3Kca;QL4SD94=5*MrJ&E5L>>LtQ6N%CLUq3*{^F*rV-6qppeEKV- zPLH4m5VPw&FJ=cTuV~&9)2$TPCQsk>;=THT!`fr2(f&24mOe;J(6frnu0h5C&B^Yb ztk{>r37qF*WroJKeZdo<(K2+*v2PyOr~t`LxPfM6@eZ3tQ0*-(;gYkH_50ipF^R-gdp1?&C4r8 zzwVWyP&o*?l=@N(u*-EC>Cq^fTLwKm6lslYfBgYF z-kf$p+rKle8UF<$48J*qbIoYKBna}-cGEwLSkI^85dC0~iHYL<=CZtkdl7?L_!8X^ zaTIv8H){vBmltpQXiwKm5G_KtD!+|KcGQ6^U&Mq5POVn>Su42PFLpjFdH5+1*+TD( zhE+YI^jjJ`Do0JE?60S4GM%)(F=T;w_4w~tua5#3K0}pTUcfUjLha>2G-TCphNYE2 zZ+$tzjpLWG##jR{tEU$2=nG}8-NKw4VEXC8m0I;gp zp#o9xZ+P6Gi6g4%(YWOMQ1V_mMKQ{zX8l|%Vhe~Lm$Vy~fPWTo|*npYJPbV_Hsei?#>^B(6K_AVrEDY}})fzcR5ej6fmA4!<2auvZA z3X!#XYCwD6q{LUP24~ibb7`BLl8u-G6-X?*T`$zP$<)xV=qClSPwfLXfrZxtp{P@& z=uO>+p&7HuWQmt#PEJmyOH*YlbAC6j)>_uMTvz0i2v@SBK-7ewT~$@bh0mOeQVy$} z?S}hW^1)hBnIPPj+pY-CsA-SBnEL6fVtU z(^-!S{I!nU#rz30Q|+GTDP>T*d)VTDz5;$8Dgvtsh7zMeQGCV9wb6m}g{+jo8h-VO zl$4!)>>BhIflLPl8KG)QL=63Qx!s{Px}w*7HxUc_1eXaN)hFIYNl$*bJML&L`>LSp z4`8?(K+yHpM6#6^!^m3YEpzU$iq)@#_rjY#xi_azc%;sWQ9}HBVHgnBuHu_+n zF4=xAz}?0L8@Prb7tb|DOQI*FCtIzdWHHSB0cFoacQ!(M?A+00QFS2zHysv;z*?si zoM6k?>1eB&UswnjudLh zSzLzZQTj_zyo7>nhrY#Y86LtGYPOHm4AZQ3mxm#d@la;1?vIRWW-H1!I1ctcxpJCf zjsA^CJ^scmYPDt$_J?M7^4#(6DDcWl~))Tm6_;k}y~157rlc*@s^M*vZhZ1K!cw zel8!g<%HoDmgeqDwU{9f<~7|(5h$_iaV<>PWQlwcF4=caY?(J7FXq^YSo8PHzxNDh zn(X_SH_TOv>;X;L*9E08 zj1n$$=GbF2C!FEZE2z(<&y>UYeq(ahIi@|c!tB9tZQR>F44dPE1Q&B9l;ZbfHa9Nh z8t%-NnqkcJ(0W~vETF;mC@H8ZZXlWr$v!t^HvjV@N9X?C-GPzNSp^&d`v{AYKRyo* zB9u~}ox(lAWyhOxx!##y?bvqxj+|$XHnTi|WJfu5HBb9lfdgzyRq>eNK;s|YafX!K z4GR=w5)G8*KJ;?LY_aw&7rK6i2N7ykLm9bisdS1g9--)<6ew)cZPm2wbrQRno6w@E zIb6YWI%@$bPv~?>^_Z_^Ptv4!>*M!j1C7*G?3jGN8ncpki$URa`KcVX94&hO4*4}l zAFGZJwp^fzm4pmNPMFZWx!v$J{Bcx2HpbEId7#=Cw-Q~a?^bz)uc!FO>rLqMG~Pvt zXD+&Pns-uJgkM!`TNd`Sr1fn?VW`gU&?hxWfG3+%mpb!bBNR1 z#p_o}ywFsrH=*~X6otRMUEQ8nW1|G*OQ=6ljsLj>S$SqVM)eB`$}CXbQYABXVL{McKu&Qlsb4;GlxL&Beha*-q1*O%Tb^++E>`4q|;% z3;1A^)KlXNCA;T{6t;1|D=v1Q<~wh;aS%rR-MiQI_9d^EM4=iwlhk(^%T!Ai+z;qS z^J;iq_Qs|Kw-s$5Z6Nv9n&!^x1w~X-N%O3oU796*HQo=T`G5=W=r?iD@ne>!#(Th* zPz}%-y1w{M_ET%!Yb(fre~I2E%c!QOZl)ejUa^meSA3N`XXNR;W1iXiRvD9m~2)icVF+EUZeF$2exHXv=eKjB?Xo-p0 zh>#X}1--Bx$zHH8tGOJ5b=LI0>HlI`G+1lkKg$1l@_C~G`&-cVr_XI~__)`Pb5bT` zA5M79r@D_=^s{$HI>r^jvC&1|#Ro3q3)Ccm%~KBOqfN47Cw9Za*f$(j7As@jj;SBz z-Gb!P^%3dEzB95C4G?dIQ^*OjksY5ihv`xAw*`Mikq!m5MjlXqm9a_vWg&CASgs~$o~#Kl7;|kYlSppl2>;-58D&FjgtRZw1c)GEg4o+_ z8UPW`byTq>^KGfa^oQ9)2*(z@K&C#apLd_tZ%ObC@7=@EV$KzBz9?<~762B4qsy>U z63s~q(l=0g#+OB(*W(bKQ78hIr8q>r2clan!`wF(QfoPP`-9_n`_j03)5q`S5}=Mg z)w=WKjo+i%#WQGRAQ8Q0=ABQkoui}Aa0?%uG1-r$J?Jfi$u>8i#2P=xvA$U$N&mOkf+d{;|U&eK0m9DxoP_un|?9(TCbwYs9ifFOLrv(`fi zO}tBVt&uJ=`(_;mT$?`blTg|nLnw`AIJ4EKOV1{@F+_l(qC-#@bPRvv{xwykn}G(baxChbV=s`Lp;~`Rd2t)`+a_YzH7Z}z00-q!Wph}&OUqZ zv(Jvt)(q&d9LQL-ZS#`Oglzk-_94g!tkJ_c%$ymnCF?@`^kQ;FNvW;QgjxOx-aPrzReNk2_4`!R<}IkV&a5PJBv{Z+YT{^^gFoivWd5uwZgy+I$@ zRmXH{u*9VUs_YF+wcU;1Oz8MV3 z$^76fI+?|ZY5tRN(PF()ewNMfvigHnaIu&h7@o&vV1V}NU|L$%_6~GfQGa2kd~sfN z>mHQ)-7#$`vZ`1*?IH>N(heUVRK8lQ@`-E4tv5{m+bsJPfj-)Bh2DX@0E;T?jq{W2 zW+?{QC7d@~OWDG->UfELSh|_&S^Jkf#+vs_rRSP$;Szb*xZQecU*o=?%Fgwto`_lN#apVQszmM*+zUlU6iZecuF7}t4iUEV9~7J$T+xH)p4dS zo3tc-HrK~2?~cvqzt^XD;WeuTEU@n9C97hkG5QAJ#N^jf}m6Nh91OZRDEw z;mcq14)zaTad2v@Y-EX=y)eAiXzIHbRZTKP<1S4mJsAv4t0j3N_hBZUqS-)t*FUK zIgB<9++x8nGd7!lJOmR_uT`1!HA~M%_I36m&>QAjJltphb|_k;TtU7t%O*m6f{Z0Ue$hh zGmbe2dp1YjZndu*j1DKKkCp)E!j`RgK;61UU4`|K1DJlixjsK>oQ47D4W?4!VYCUF zKZh0V%(#op7hA0u5I?+K;+`iVc{^5v&X@!{Up18(ku2)?0-d+PO-Z! z3i9&fkf_(-S41(Xg_K4d^B4D2@9r-P8oSR>?`VwwC@U;Mu}X zG>{Mtzjz`-U8^K?gOEB$<4jx|*rxOjV6|lavpW^ufG=PDOT1CQZ z;r`HCyXL@ew`aSrTM}INLO=c*MvG1TW(ujU=LtK{3dW`WsJ62R!yqcAcbRP^tarBS zd$HA{jVn-9^JZMr-?hcSAuU_E@4ST+Ev!mp^iCk2^THbDn4yO6wCMd**vZI?*52^8 z$NhFBl~-2ADNxX^-Wh-!KKniK`j#TcWcB$$Y4Z{cG3&Z$(?hp4G9_ve0RT|rCD7=e ze58;=#%ymtUwuumsSDQT(0g%Q`iM8iBzf9nTCGBG4f7mHOK`~PlJ7mgp>EduI*L8V z)HH%xuwQu9J1@Lt!?EER4QTB}Es@#jPIHgm>M8C)N%KlW9l4SBlsS*rTGXrp3RW5Cmr#xGH%yiJ*A@{+ zf>R~6q7r!>+#bWHy*$^TfNX}(_)MhtH;T;obZqUnG$HCkB&Yc&(h+bD5vgK)(gr#m#)MhzN#~9RBtTOu%zl8itX%_WUsTfElp3<0R?%$-mQRITlmo)#IdsHP7ciXZeE-WUD>{_ zf(}O1#Ew^iCg5d{xr+c5bJFG5#Q{x~=wC(|69ktyfhJxY?jhEv*O3Hskht zg5!plf$Iq`Cro%ftM;tbUgj#%BsLlw^b!b2l(P79)WW^Pz{<81Mnq;(rNIP5*U)e-Z1>Hz`MP>$1v;_5{`3BAZVCO_|G;}sjjwH?sSQXC)(FQPr)PxmE zn)a|)XK7ciQPF#2^@DdYP6v`LUti>Nh4Zh4FL9=eg5L%pf(e&`17Bg=e+pRml3zE0 z6gYUPWjka&`Lg}(UTA=J@qWL^$Uw912|65#4~?9dorA6vxt^WcY&pUjwECNNze#iX zZ#_8&F_%SCxqPEG=wP$%+LQB%Yb(KKUrx$(DBLhQfr@&?+jYUIq?1@o# z0WtHNrpKaXZ_|9nlZO#Yak=d@)u0gCu+(n}Ez)vej?8~z=vF;)RD78^9JIk9pY$cL zp?ROpgSfV<#_|!u!w$3xg##IKwD#GenI0`3>E!{Q{Ip{uRSRYF8920MajK7BsiMcb zrO=Fy8kOF@&8)KD2Te}qwd~daI)q-1jI+ERy)$?-SYYyIum>FZg#>%+$R7x1r)f>M zw#p8Oe+sdmFmWy3N!DiIG_K12^wBfGqrn{aYSIC-*YlpP7b5sow{6fyy(8x1VPCj5 zMvbL4J37=Gnfw_z*UqUY6k7!DXx-3Ve5`+Ca+`O0rwSIOhDy#TcWHdg>lUD6;vJ9Y zwPS=>PLg=NS?>uFuF$YNxW_heSSm4X&Hci=s%7CLlC}zP-;Pr53`kx?q@_A%!J7d-X&c&?|5nHpZc^2x4nfzX+Fd6$4Hr)yDbVRsvUBf{lF! z6<2*wIW>9I`VfQD`@7@w5A(}9Ji2&ZEXb}t{xq#ws6FOZ$^Y$3h@IeQSPd>$-C+NI zG8TqMQJ|B8_4JaIhJkS!M%8MDpCSQIaL$xNfvPC-fj7`mCfEDGqN2qOGapvy&6^^a z66OYt5O6~4^99;7cqc;}y<#oUdI3xd7eRI-^}zMZ%AK`FFDk?e8EoY|(5CG5*Pb`iQkKwa-1uEzJ>aS*T{8 zTU5MgzvXSuG!xxBEiN1 z8dh4t4TH7G&pX8?(k<9BCdEd@Yj-}+d>#lF7|3<5mU0fJxUHFqV}AlPDf4R*SaSi> zrb7+Xi<^w&j6tXrrm5sat2&WUF( zYcdBhE-BVy+^hMtcQG!W9eU`zG^)CScwQkA*5;P?Zsw=pEMLPV6*AJE`r3EM6c2i~ zEZ%=@HDi_U?GfjJYUcAZphO6*oIlA<4Ae=0SFG|v>3OY*KtB-RzgMVp#V!*H`te{` zl1)Z@F?`ZX|AgCr)aoEM#owk_fz4k)zzEzVIA?!aYHeehi&C5_y)ZP7We5hBAMSgz@2?i4#Z_05xp^HLjx0uawoGenWqkL~ zkf2lwy4~}u42Ee+&WuO6=>K;A*Y?Drox*lh;>j} zDOyeuMZkJYf~fx9RHq!)oi}Uy#(8s$nuBCq8Z7<{gkt+Kjk^osAvcNnE3+w1mAD|9 z1v;}#qYl4P)%jMd!8(bZE3fwn^_hf6)zPC@5gb~~o|t4dFbDaO&WwJv;uYynb`d z%NfqSD4qKR7oYD={;&MX?r;#~5P~q(XxcZQN~D=XHoEQQXMjj}P^67Ylkg6KO5{xje)oy@J+h z14UIhVyst|Kai?gSyv0L%`Np2I&hjqog1D>A`L%&wQX|tO$p0vA<}i#HGz!H+H#oy zc;rGuf;;I$&W|K0gq{MCk|%>?o+%20%7cfFkoW`++?4OWh_7xoH`1K^Xp;qzIKrpj z<7S*O8Z8HNoK3xRufq5D4=zB6Ldep^ZaN}OTgJ!4H@JTTR56#InNvE|EQrd~^3>+T z6t^F)9d0rZtTlfVIXH=Dc(L`mv{zSnYx7f9=5feQu^@acB9Vw(!Q=!Ef^PBnEZx7k zh2gB|bqNP=Cp6-x&IIu0DMt!ltM-Jdz`d6F^=e(>A7qHm-pRv{b8J_O(P}P@F$pk! zK$h~V>3Ik(&?7I**H5J0`IF=x42Rozc9OT{}tv@z7q}Bw$<_iOcVv4=(&~{Y=Q@F_1ElI2JC++8~DRJGX#$ z&TdK`PVEYhOpEJYE7HAYU#M10WEyOL^u?&&3s>@;&C%3BQiccFcK%8_n`RM^ue9e; zs?m`?aicHOU3CBcERU$`igkXMy&!Dq`;fW*R~7bstBs|Wo40`0o%Svzk{kD&1l!)0$oI!$^I7*gu{17sRVbteZKyF(5S1b3scp=Uc&}}B z7@Az!Om-n*Gat6n>Oin&F-fJ9|Hm>I&!Dp_iw~5f;OVq)%iE~BHI>gSNi^`=ObDW< z`56+#$qNB0Tj*oVQ-qg0Oy$~OVaFWJR+h~X_*4RTXWz=|I^IkiLj})}Yta+_BleQ| zqM*xjZ=eD;%c{}Q)H5PDR!FUwzu9VNzyfV8SA7C+xCB(Y5ngC^V>-w8vhlQHek;Ox5OY_o>M${FkYAj&LB%1NUYS#nW`X9j~7iUI(qp$~^4|sO1qinYNYBN=kwjbiYQpIGu9TF=n)R z%osC|SSriyT19t!e9`7^vp3WGZ~S@8dllwAX)^pkV|PY^E^eax{OElv>$CRwXJUqJ zZjTIx=>tYcO>}sfKGQ)a501hiKji%IW{N;j2{ z=ChiUb1`g5vYWoOt~wI$_#F3E@x#fTh{~nu!qV7_-mK-IN6mzSFqg{G-KUGA{N7uu z89w6?BVJ{jFfS9&O6{Jkv1zO5h(H%>@{0fQap&$3k~Znp9=!jDQ}dpO z??pt%&dG4DQtn5Jogpd+=COASXGW?#>*sgmVK9$Pa`!!({39>3g_*v!+{le6s_FnBoY_C>!hBEBw@qOeSV z(q7ReS9?XT4)9K%^{8tEHzaHVeybZ}u~m#CZRNLEh?GJe6=Qutba6Fb-it5PObErN zR7@AC&ZenU4G<->F;^Uk*D=+Y{8Su@X|N*avNBWq{(i1b+Het8+Gg`&xnKUs#`c?Y zyZdNwXV_Ab?gYwaSMzdSqvOshzws-#-3*oNPT{*Qyr+IA3oYl)o-p63@IxC$IQ_rJx6WK$ZM3ao&E5su%eyG5zmv z23Y}#>~Eih|KscX-?_%9kz;P5QY62d-+TZ6{Q1v(b)raM`a@R4WdGNpfTk1DKw|EP zI%UTHR#5W^sNG=%s-*s-$mH+BBPm5x0`#rKx3d53OMJkrWwRd}{@<1kOjP~`Di?c! z=e_)Y`O=#wz^tvR-u%oS{daEupGmQzs&^C(nQ#5K5}zSp*0&-_o&Q^X0uz>t`n~v{ z1nPVq7{M>S*PqjTlOge66NWi$aP&(?7D9>sgQ+~O%chhLizTiUcj`E= zqUO8K4%93jHr1p7U6^x{qz2$kxFkMSk&O7YsbTwXZ?lMun6^hE+D33Zj0F5Ti7dtc z?&=8qelnXBgiWJVY!%fL1&ge+Taf;wAl9e>xGCyYS-p2148^BXbZ~I!Vy1hSZ9iKl z{BBYfB@*vQulvbQaF)FL0avu;x1J9ZT(X~BN;juOZTiiiA{TQecJLt;O{rexlLQWf zl_WmvVMbOvc`S^#w+8LtGxe_Nu!&*=odV4-v7co^W*jf`IAR&6`eLljfI6?7M5)Go zt=lTsc-yZll==o&KB6PR`l-(}JlGh=6tWQ@|LiA`IfnHv3dzUA18M*>J}Wg}2L?j%M_{ z%}HSmgB_bs;1K}Ejxnp6you-Gp!50%L~+r2kXuT`ZH=fRP!?yqGKd+f{R)5!0ZX(rMu$%G7eQ-P5;Hiq%O43>1qRKaivJ)=|CX=y2XlQ? zZ44&vV9T_}`&#RMt4JrmSj`}y5p$Oyl5DS+AW zMh>EQU>xMl0?gLQ(}@6((iQ?hz0{!qj)K%l2c5+mzaohoZ5gJ5hN^Sl)E2Tk8tzEA zO+(wX=S2SVb+_)TLf8y-be0N-*E#16fmq&cpuDN;dttvYL!gxUjmdBLsh_z;JNDOx zs(n9}l=DTkPq@ifqc^>{U@Gl9MiUaNc~~M$?e`+o zQ6-F{b*edlS{_zzk&uPH!rNJoJYR6#v720f?i{?uyQ!-CZ_SpIg?3tSr_qX$0+6Wt zFMxF_lBczzYUqrE7D%64k?93}-L6!M;g-f&OUVMk(^3QKC+TNV8Z);0L-UEDW5rgN$_QMHyUShq@zD%lOFB-~(@5 zNw=qNj=K_v8#N8-wO9I%hJcL;?E7y)4ALYmJ7pBdp)Q(9Mchaz9rkKa;X7qB;O2O^ zlZ*VPFp;qtjYBH-it@cQAR#j7JHho5emJ2}Pg1`%mwdWbH_NfF=Le+zSR0p$hz+9> zs2A>IjOd?S`G*>GF$EZKfE5!cgHG8}Yu^c&++UCvE4e)uE+L!60e2= zT>l^@-cgz-ZEmG6)qDuZLszF1u)AMhU*8o|lw~~$adsVCet1xL#vCVePupf4vK5847jyl$KHY#7{nDptFXNb1ZaEih z-VVwk{-sb4NB>c%>I>FAvsI5{8NqrBtNrMJVZVf>C}NTu--mHv%;Z^f7bvY7)}t#l zjOr3J&+ldRnOzmM8I|vjXHWcKJpk!>Z6>d0Y@AVK+%&DAc=Th8iG{^GjHPQfAra3k z2PpqhH_Y13aL;vVd$-?~#Ffsg2if!#z8KXA&WLXx3-~GxT$7!Txy`#ZXOFj_)dyQw zM}o5-wvE7{Giqg^N<4t*Z^8g%CCm7(cO50~stG|-CSZ@^%4q#+8xmU+y9g<^>qUn3 z!wMEaaf?#^D~i)R%{mo%1`JlNr^Bagea+Sqss{!rKnJ=E41Mbdh05zXcf$|U8g~wC zM?cd~Dd>B}iF|tBeH+#AOr9RSSa`|4BHgImZxqxsyggsIn3k)hhmXjU3`` zv{hI)6o9NaqWesM8vJleEUeE9ct9D_Xc!bsz>Ev54(uu1xrOwAbt?c$kh`3up4c?- zj)JBT2Jnx-3&3{0V?yWDO8}O$ryp+P)$Szc)qc~Db1b?6Gp?nOFna?FIaoI;#U;P} zy`vRq*}?cUkX!23o1PNl>+~(;bo9@4BhWCcxrmZ>%30Dr`;nNw(tQDd0A|%;M4+o1 zew~8?>by>AGPertMo%jsvu~cY)GT%O_d6Q|ua#73g^hDoTy6S2Z?5$4jg(W0A?LzK zr4N*n#CDQhIC-ojU6j)>5TnAV&6hpv7Q~i`p|5eVs%6C@!r;5O53V33GWnHI^ zb_yf~50IQH5ClQ8p;GL$`cwy2jylqTg62 zmjss?aT00}_9|lBpBhQg5Kz*$SXKi8O)9mAbx1D}D@_!6TZrI|xXWCt2q62R|C0Sq ztRSm>Eq!hd-Rr$N*7VoodtND&qnP`?I7(R$WH6uYBdWW2bTE@RLh>2bBy<5MP%-z`qzKOf`6+{9<4w>tvqAzen5Tk zJf>sO02<1pbjt9v7KtKTGHuGLt|RgRa(bg)mz%=Fd)=<8f1M3fzlqm(Q0L0C8(0!WzBp?m)*M7P_(|n8aqjW_eXP*M5CMePDWfC=+(C z77Yz}Ccrp;AwF4eMK2H7pTWg10bBHMxoS}bhMc*H8t@D?p#Q`NUTL%-Jd(k%BVY^t zu4wA7(B*=qWAK_-h3x+(&l^QxshWneC`go8V_<)Bnq?VLJ-y!;Np)Oh5ixp5*8KbK z1=z)esIkAW1A_6zcS{~s8BO#)kQ&PfN4}Qhk(-m-uOPthDN30EtGefUj}%38J75DC zp(OO;sL5hrVbG3O)4c{>_xsDJC!%`bWbQ0}gF3Gn!vR~5sVAO}8ubgW0MHVcM_Bxv zOu6sh%aEeuxCZROXUOos!j#P)=+#VK|w)^SQ0*bsGtKu zLt4oH3>fmuDKlK^S!vNn2}Ed;<64+iQtqhUS0a*U06a2%>7oRqLNPlHY$I75V)XFpJHS>%$=0_Z!>vY)7z7}l#RDb-;n`m6F8{?PrW zkH)`O`)BdrQNeXMTiQ=W@qZZ!pri{fz$Y^kVWa;v zi}*8%zu8I(57?~lf$hxyF^Ru_jF~}gDB_6T{qO(%kJtTj4E*sS{5{C;DKgdqc2gG! zBv1V>UikN${$D52t=6Ni?(TOl*nw*XO2Ywv@n{6f<@Yvjr}6)5qEDw$#B#aT-9m~> z1^u-`snDCdn;$-sWsLm99QtFNDe43Yo^qd9#(xA(K0DOO_mg@Q_haJI(}OJgSQ!EX z8KfX@ss2=7ztN>*57#*&HhoM?Ja)fq=CiQbk`eCjo}g$!P8d^S4xx)Ls8J@;4d8AtuC$yKa^trJcZvm3!p3 zV&v6N{)<|lNavDSZ7Spiv(OmTztHVfxb)oQc=W2J?D}tV>Pwk;C9s;PzGj}G0MR<_ zU;ivcT1`>^=Qx{B3W-HahlgGu-TNS<@?|yS!PDS8|e$4|8i`(A#fTJ#~ z#Cp2;8hy&TpvJvBufEH~gMM#f!SyxF7zskujDntH)C^D7h)p?M{?7!J$(-0Cm)DZs z-tlKVGX@q){Ebgf@iiOrZ_OE`Niry{K62~(y7c6oS4*p@2?zSl5fvkvO`m?6$tZe$ zB7L}+z3yweO3R=j+efyD4RF-;4vT*z3nUTn`8O889|7XNKACX9ZP)#`hysqaXBz*% zW9{F5qJL>8ssE2U)&O*nw>`4={&mUwB}TvGk&ZGb&FbE5L5!etzpmK~+DEdV{_u+R zy%?t0c=I1)NouGQn&_?6C^^4Zc$ra1xKdk#tky3o8tR5N{}wezl7tC)4qzi zaz@uuDkF>N?Hh{cXWv)N12#Uv*p2Une|d4NwsV z06gwJ_ov*iP4>Hejn;YIjOEu(#!O@Av&vqX+ryaEIXmuzJ|&9nWbY=YYa8sFtqS=T z{qm$7xPkv%d`@YL$HBv$(PPzFjip||K#&eP=P+PBe>3>b>i64(F{UYBcOSiNh66HZ zQCxGciZdggaDTdAe`=|+0 zzSAj)Ndq`v=wz;Sc3Q55tXBQndOCgdeU+hik7+V7Vmgawl1}dg){!dz+39(#G)|dx zHkJ98_G(P=Vx5Yw(Je@h$DM>LB3 z>mNV8dx9?i%hLQsRY2qY>nTVRaC(Hu#yQ{p>g2Z?e;w0p!0-HCa+W@5^ZRcNj2@ z2$d%u>$Y9R(LuQ=@~B3-bl zFXO_dS$0-A=fI(2mgnK zzHsffrU_TLjIkkPYf@>jKjHTzaK#BrUd%bv_=65Z?r*m%(-EQA_MuP_$}tRZDQS=n z+^gy7yqvan0;7ST?tK3i?TC6C@?pb%sbuS8wN&>#p&hSH274)C;=i}JQ$7fR)5t(2 z=;$)g1cb@A-Ewa4P0HnT+%rBypU1=r2HxKTId@9z@Eae+PMI zp*;E}%PZ=fM!*1N4Y&cQ92C{fZYK!_lyE5k_z5YR1-f0e5y_(@lldYL7xD z07}85#&IIg{^ORUfT=8}Ud4AK|281X9FJ&^pkgm=I?d}%7MF|K8q=_2k8;oRwRH44 zu3wccRq^nsF?2sA=`YYx8Mtm^-ujr#kZwxPeCagN*o_o)bzukYK0V)!i#D%W*H;3P zztY%%#A~@%TMW8z{hhc+GT1$cO3R*Jz1eV&Qup0QD!EgSaC+v$sJADDMF~zEyGOqO zsd_-j2hxP>g!>A{Jj-s5;Wq8)KDl9{5b};BS8pUZ$Fw8L<-EM3-Z@#GI_SX&wVXc> zqbC4#QDHT7kA)QwiVubd*@;CjknMe4DSX2}ng2PKfc-RMYcJ-k*+JRKQO?_njgso6 zGLNks$4rTeQg0#qvApWuw2LO&ql4%J6!gw^WBMR^PZ)p|M3ZIFy=1MC+vuG&a#1Cb zQ+?O@`#dBJ3ZN48ATcGK1e&fs;w;bd+ip56)ucMa7HAcAae_DR0XU_lU_+M6^9`;c zoTeKl0UXU7NEPjrQyZ{mqWF)|_p0e4UEJb6osh`mJ^RXY#O!htA>1=@tWZZy^@r~( zQAZZYiA&W1n$@Z=HK>XYC}ig?>^L8w$cJjVO0x+>VyPI{omsmBH7r$oK zZHUr1zdqb_uHp@N?Hwp~7$Y;|{*n2WxNNCWttwm~Iiw4~5}L{Mk4-4nS4@>LP@3SB zQ@l6}DGl0F5NE^(d zN?)p|r(BR9tie53G5%XK$)1pE9camMI#wMf87fLP{!_JoH<9X)FRGWL&W~x`7Juj{ z;6BY~?fd4So69`Mf9!wr@JiMSHQruNPw5t%tgow@_{dlz;o~T&wQeUx= z;r?ywo1Wm&hQe@#xzdfA=$7#V0H`U(m9tUG;;JlrmvOS6OXLc}doIH_7jPm&#?-`- z5r)2t_}di&Vfcw(@}4f$fOlK~h@P`_@4Uv78QdEiQ}qA%zkLalM@Qg@(@08wo{3sR z^}IwxV|4Vq1?FCRVRNpTaA&wj!aT)Sn1s5i88ElJc6CUhK>Y{g+5i=~+Uvkw!`K<<9IMAHQ zu$9FQ&JXeeUs$xLzemBwoR5&`KfR@Z-f58cunkEBGAuQA`4*u1p4R_-2-Zw0+vH$# zM~ko8M#BFJbKOHz$1^>B5Th6XJYoU9@!m*tx8CWa-@#Vx{x%M+y!*jJ(WSUY!yk=& zJ)6e&_l*-*`cmF2EU`dfVgmRWX}9Ccspjo*gto$CJ51qf+I;)Hox6RWW9BtDlb*v_MW3i zh+wAQ!>7kHri9>ke>+$KpkMDmAOV*HQ}S$3JZikoad1uk1ow&pl8cXhF3EOF8x z$GklCSyUxHX1Pojj1?{qdf1x**(E0nUY1qEE$)J_`?$*Rh12{@V!)?KqMxFDXG7(Z zUUx9@f*MT26u#1zXkrW|kxE^LSuQdOdfGIhEm$d~Q=47wPkjQ^YvR7Ly|>1=R{1=@6t^`h-?5f_B3X~BhpZ1*YcUQq1)mmLh?lbe!_3L~~7BUKiI z+6?fl@`N#N>Wo@gw3A;`2rV2xPbHI?FbPBYZ)YnL>)@(TK%#WEN{^)@ zxggx6?&C(gTGCwPz3eBT`P3b2eYVLguBs-kt7`14r1#lNNNe6Di0DpilQO#km9fte z$q;Aew`&~dLYI?po^COdfn*5{W1q`wU*^_rlqP0DvrY3J#2&EeBirC%XXdoQXzWYO9?nuy&!~Ajp?%ksB%nbkBUUi91828q-O2%s>B)#wE zdxLn>grAG-Z7->cUmW)84LGsARLyz=0IHy2p+9<$cQ3Ye`y1z`{}H9nT1I*@@JH+l zfy9Ry8O@J1zuPMCA*rG{KiCFiJ}GRE&Rg+oXQA;{mZ0L*&hTDPu>F+I8xuuF*Yy|; zQ{E+>Y+4!(gD6l64I?_W>4m%0Bq^*yQAe%!+#$t%?oegIh)blbhswp6SzYpl zsZusJ{!*J|W-VpyPNcs<6}`>Cb@kw$_=z2T=3;zlF#SZH7Q1Mj`$hNNcO^R#r%pNy z8kWlujjJ??nSv&?OjNx*i=`}nthU-Eu`kGsH+`1ubf-50ZsgIg<%rmAEkJ1IZ{M}=J#HeEdIs-gtu$m(SLWwp_)=R22Q+Ha5Lhv-SrzV(y4M_{mH zY3v0z?;MGGB>=HhLq1|oA;>5pckVuFb>u6I(H2TKL^Kr0ZEP7aY*&e>>towUxpW_J zT~1WXl^#$dhyu~D=+Mxw1H>bGgB|opjv%(TSG+Zj)}OEaN2Rs?v?sE+s27o-CP}sG zJj}}_k$Dobs)tXBJYIVtzTr_Paxj~q#6`gj)P)qS9YbKfXh{3o88+I+t&G4UN8V=~ zF?S`s-b4TZI>Ih<#1=|p{XCnW!i$4kER$F=z4f(yw;1RAOkhnJGJ9uArMiWS?zoEw z1L8VWYU^p|N6ix^f>0%ns!!9}Hss1DGtG{&CJUk1Nmk)iX?rmlfUl!ZT{_b*v7xrVe8MG*JggDh1x`grcnOEJz2K_W z#=d|L(A4Ytnm6^8|1SDKC6E{$ zDX}6YYH$bv&_x!~`d&6ymQKLDvMte!5nSmnEqdLoCg48B_=e&@YskuhQP?oB@r5PbuDK?GEz|cn)6jR+-_l~O zLwb{=DJ7{Dkv9DK1*Y0%+V;np!WV0mY7+6GV5+Qi5sz4gX!84E02YHKl9l@VgsVh9 ze}tABc0%QzFI1&9*L79gQgE%FEkBiMyK<%X_%qK&hn~wOcZXL-y5Mxo1orM{9h10A z5b|w8xrR;bj(oWR*de|8D7iTC$H~#_(jtO{t-MulzRxDJ8i`|{5fh?2F^MK#L&xgN zXWaoNd{A_}qq>cH_P*(wW4pJxJ)4OSjBNU^O-(F@QT=4ZUpy%rCv|alv}9$9b-Glk zR&3^W?R81T!~yL}5@xwl-yGy!#s>G!>gjma#R8L(S+ejPxO&~h@8F^~<1+P3bmNd$ z4D({~Zs;wLXP?PT+?!FANG2PiA&wkPiK_)%S4hXeIB9J{@PuS79<9UCL|VVk=aVdk z=nn%Dv*txZH*|cSyg}%Z^gj&Rb`$`q^WPDAj8r8{4+vUgBOyr_J3b}5EsIa!GYQU^ zB9@o9*yJ6MeF)7I=kE3QFs>drJWW#TL!Y~*y860-cGdjuh!)qe*<*ttz~}(LNloZQ zh*cjSoF55l7=qZLrN(Mv+gJkv6E(=d%P@nBCtFGdR z&cXgAsN_Mk78i5YcWraz@b0eh_)p$2@@|XwCwwq6C1ctL$*NhiA%vd10Iwevu$MSV zbdN;oJuV^hr*UIe**lBOm_6b49Ui3<8L^I74777Ukie9N#j2DD6q4F4h82SaC>T2(wr%$Ml&g+-BwaGE*~N>8UCehn1V)nKQ5@24;&_K3SBLZ z#;NkypR|g@^S1phGb1gVh34zTp(cGig}do4LBcp?1A(mt=l%k#HbUm>!{+^R15G6a zLWfy0VgvoDk1W&^HAo+vkDZbqHmUlH>;_``ozc2|5RGI~BGuW`DX)}X8OUVG?mKxU#a6R7 zAd7Hn!-2D`_Ro2~Tgxu972bs#$zk7_=Q56yCL2Cc|D!`&xPn5$XCj-k`!+pHKJi8z z^WwZficD@Mnvm65wD~z$e6YftGN>wF{*X!bdZKPj$gO^Vv)c`5-PlsU%P&B8_EKVu)@d>$D#B+FvkUaLWPhkQjr6?XX;t}EoOPYQgZ7)XQ~Iy( z`nP{rcT2v1Oeo;iEme6qo%k7jF(#NGR>35|zDe73(56zX>8;M9p@|1i&uEhl__J@g zMc>f^aU^EY|7ia}sRuXkB+hhoTr!j5sRUED!kf?c*`s~0E-!Mmx27|+NPP%8lP#({ zlFEIV)z)ju!&1h`Yt|Tr^OZBXVW(>d|Gw#JiP`Fht3*Dfy=CG*6fyH37}nYPvgI2* zL~tmJ!3)VZo%&K5bk;?YEz_lbOM4wlmseu<8?-8GtH-Y&{{{t z=g9^ws(~hi%HoqqBh|`)lnoybi4)OAW0nu}T}r^ozn?&zB7-{l1A|+D+Y`0g%LFPg zRak!_FbEE4%d=s0igkMTXyU~E47pVs-`|n7ty5&kihu`XiXmpn43%oOT13E~8{+Y5 zX3gpa9A-N63#RGZS3EDd4va1#f;dAk^MnV{&P906K2D+T4akPsW=k%U{u|2zv*fL5 z!#U-?HSZ6Mc)2j!OD*z}xiCA8Rs1^Mn>@XMH}oIYh5b$2oEv<$@;pE zs3YZZQtZDUZ&YVaN!L~01$`%Wc|I#qVsu+912l880Y2TOB+CbUe+|PN0tWe~w_)pF zStFI3c3CJwsbYPw9tP$ZC&x0N^s===)-wiA28#pjEBvfJ13{#jW~qaCqA$3{K6e5# zMwn!|`ubQ{E4%xB2Odw@ILCfYKF&1$!@}DQo?8|KOB>96se*A|Su*ClY1i_2vWHhU zV%0qvX7cmNMH0@2@{1aK%{RMvAJ`2A;}X)k<*cUNbf1v9h$8pOaa*71Dq2}H*A9O0 ze!doFgLA{{$o%egaykMoDgh_Y1jU+qs!i2+#dqQ9(V>x|MHf80rfPD79*Z~sQJ6mu zlptC9EGQuqn+SI6M9+0RBm?KuaIG3GE8Bw#ZI=yYKO=OA*%XiWI)EMhBh#%Bd^K9w z0pYPf%c#7w14cSDLmBJu?H^>F6&W-`x@#amKwn>(7=9E<;T8Qx%zq95A`eZMjEkPf zGHoja98-MI=uIY7UF?8wL3=uA{jAan7b0ZpvDx*K-@*bu8x_{9QN$_r={;og8Pj^` zL-Vfof&LFT$0xao61cIKsx!vN$jKZbyb7G33+PgkBBb=}R+GF~@;jW_s4&@~I6hVq z*X?Ho_0BH^5yX3(;V)mF0{%`4NmcRnJKo!0@^eZkCTq>}OeHm-eGm&vFjcngL`kJd zeF=MC#hh&H5o})p*<2-mYj$}vj1GL72ZDIbUD=;N zR@EYS(;$cRl5L^>q6M#!G{=uF6XKo*Iu|U!T`T43it|H3nD6Gg&Szu}tA^jPT-6T0 z^RP}v7MYRL$2AXq$kNe}W2e^2Q~Uh2Zakx@oexc-ZcY-RZdc>rozAjw&XQXz@e?NQ z_3l5a3a>=$+wl^S{5mEYZ71bEM>aam-6V7vga);D!YP?-?itiL;Ip3iHytC$rMkPj zml!HdTLv{$i*I!lmNu=pY`jAM-o`*p!!tr3w?qtR7BnpNOsQ2)WF61xr)2!9u8#eA zM+g!C)Vu`ulwJ(xxw99}+exaO>?=*Y7_9+wa{G7p%KNMh$ra7k?ST_4pgV=&%1(GH*#Ch{+Fe2rE^QL~OTt~C+ApDnxuOOZTgK*a8g*?A`_nS0BF2tsUyX}_El)fbIIrad4 z2Tk+it0~J2CU>BxMa_(zcH}??Z-;+X$(i5xa=D(!+8MhG_|=+Bb2}lv6O=k2-mw|k ze>?10kF)s)*LZ?ajj5dJ`PyiC_67O=QRsZ@W9XUcX)_LO%-g&Gn{bD6_w#Mc*`B+W zFV>Ke7V+lt37)0b7R3uMFE7K5FC;dwIP)8?z9RQKQusev zI@n}_Me2_W3Jr}*8qe601qYhacUu9c3~eO!;$FVF>*ljyv4NQ!9CS|=rD(A@rJtsx z{3DEpbHr=<4sVw{oKNZQaN!cniU7gTXeCbY=wb)*2?E)8Py+?9Sc~2xz$RMGg4Q zIcjIOyTKQnlZBq9nutlGIFZ4rK5p^J&u&A~)>7(oFT;$y-jgUNT8SoF zZxqq&Y-J^s>eAb6x+tnp?!WK!R9PI9nZ_X?pX(YPXge$q#g;V8_A!$d(0$tpt zI_^!Xgw_4L@cueq==83q3A`Z)`l-?{T^OKkR+=TUFb)x4;%Q3erkQND2r@H*5sy?(UZEh7C$c zw=~iX0@5wDk?uxHy1UuJ8dv5qJ9e920y z(iZD^HaCHzsIoU|HY;n+ZvMj#?f3`p!PXdnEvY3bM``570cGHrWo5VK^m6D?>i6}- zo4A7f0{ZI7@&6>up?}D9WgZUcIE=DfaVEHD= zFHjTNBaw)_x{sQ?7nUqc0!uk=!e=g2`- zu22ss-*~6*$igy!bh<$YbW?Dm>)<=6SvK;vS55m^V!Xb}QK@O7l47$-ogvZI<#iY| zd$i}CeTRXdN%$01W$6=g&cJ6;@U|a{^`V-@s_RUi@+O?-gQ%EGW!g(zF6+YoRm^8+(^?;xl070wJ87(|211 zSP@}-hcDxkZfsIF9?#ou73<5$7rZWfOevZC^3L;uVu&wHA=UGc8qekYL2mR`lgSy8 z-83v$eeKz%D@AGJf&cnNnDoWb+xYGwKXahcZRfkvis5ssX_oIb`f2uhx%LMip7pGL zyS3m~kIp?##@29Xn=U5XjWCem(5A1wmY4hXW+!8Mi8s<}*&o#@BBGQk0f&7)%a3m9Z=D-9Z^G=l_;2a!;M`36@$BQn(dFIPYia)0`32{J#&O+7e7fX} zn+l}8JAT*kB7GYx&%(Nfi_NY3GJZ-MSf<1RDq^obvCByFUSOLzWXuQ#(5hTt$X^WH z@sa>}^^e&fZHyJhdmQh&59d1Xl`ZF~l%{S(her*SHqzHM##X;1n4{L;MXL~U6dsNS z`2P5nvmNR^-y0#X>)9^z5s{fE1Gi|lLWo&CJZ@pR!ws%SJ{J@IXJ#!K2R4 zC9RN#5e{x`yTEvNFb=cPvd=rukH3C38}-&}+uX#k9zvYH$!XkgrJ9c>6_a|{|2mJt^fYu%G){4r@>pQw|ROJnW( z%%eqvo{U?cFQd_h%J5x_#@*D3#xZEVn$X_hJ8P3J6^o`kY)HWVI=o_X_&(_HpnmT= zP)^+Fd*aFAF_w&KA%E)Y=)PnNpn5k7cTK|`U_&|wJohMDOKB!Z`BA02%QIDJTrARz z+I+WoaC?-TrfFLcHg8wS8E!DVhE-&zR%;scN~=tEyi|Q>3Wwwq2TT7*h^@)(c5NK} zh@_78_tp4Xezn!(c@66;)=^E9Cn3&EwFh-ujY^fNYd0KStEFnpXEfw#P#Z~Hcsye> zc_UKO%&S<2Y0p42_llFik*EalC8uNF%eh^T6{26JAZCmY5s1(Jr{s|oEZ)laiqbuN zRCY|O7PIkqK91py2X|--9vm>5;Ea z@8=LlBVzj>u3E&MoF>yj4r!4|WO!<06A7kU4O81!4R)*IvnO92o~WK(vT zW-S`J&@T;D%jlD;)XKHRy{^&7q z800#4>0vHa(|_~uUf%ts?;wMKCkqOF6#h>~<(~t7k0pN$SJfgO7xhNwpKSGKGrwL& zk_|3WJJ1kc4d?}0)`E2(h*e?=5dR(x`H1dvvKntBa`8mQYRUFNDV43&-z)et0SOfF zwJFk-^FUA_100Uru_r3|&-*byLW(ogk+acy{yE|Ag9LibfH)Ay8+(#w|7rXEdHB~Z zAAp8^0dZLQ(wGVPAD-|3_fwPs9v3oP*Z`yZuM_&mVg30@LU+HtV13Hy|LxGA$E3hE z{%>LT-{AjF?7yf8ps}fqSS>pO6sRKsyZXF%RV?B%AwMJeV3{^AbuUAp-pBle(eWst zUnE`ZAtARsv!7UEVZ*FuzLqLEkX>(_tq(AXcaf6{M$U(7Y?t4UF%#m@ z($mLbGA|3O{9G4NQ}HADj4|2ZSlD$gCo4 zV{TaKa~;*0)I^cUs|5{7ol!Qg!SIi`Qg;Ue@!4G$NHU9%LP5+@s z6F;uMW$#koWZ>`~{0`VYN49X=d#83c+|FSR8EyibCa^TGdBC=2dnVq^`jV7=bx`3K z4<)vosX`?MY3qwYs3Pe!YX%b~bAT=Y50F!C4;NHw;Teg3wqcwhQ6h9Tf}^zIP)Njd z-VebZcyDKZ`Go0eo(zyC&ipnC!WKV#q%_J63O{}>v` z2pk_HqO{%rCh+0FLTLc2tus?T{GXKMZ?&W^`2ayQ1RDMPUtINHy6VqbhNuAjaTy{8 z_se4VpB4zk#cFwgY4C&mHzN8!Ef8{ytotB{&p;lK2>;Woet*{2KJB<1FoEAo{U3(@ z-?9B6*uN}-PJrj7qVbWvk? z^=2jEu;?a41n zcHRq~XKGd`*eU{kf>G%EO3gcUUPpcG!Q-}@&QaKu=KnSc8DBv7X(&I5kqdxLJ*%le z%zP0fAAIp`DO#SY+IGk+MUtm`{|G?ABlWyIJ}|o;taR{E^*&qhZaxV%ylj66HN^zy zc)hz+Z)Rp<%3BzEg|ZYz4Sjl7D|D>lA{+22M?(KM_dy*f(idi*q|F-LoZsfWqZ3W> z7zr0oDJQ3bB2PMMbs;!t_7dHdcOf}EM}b1>Fk2o(l~AO;xv0A9zGCt~A>x@##3)tC zog{`wc_B?ROGOIZ+^p8>YIN*EUy^#iW=NjyAAU`5S)@}D`Aioi*7-)r=qxoyl zctDqO^OY{r4)Cp`j;x0dnGriVR5Rk)F5ljJ1fsI!VV}^!6Z?zB&qE2UNqMLqL3QyN zoA9^iFBQ5PgZ7gpLGXC|+a9mM2GKwMM9uOzp_?JZPLF==6)W_?O`Z@f#&OV7sLiIE z<1&IIVwkcShBYY$aLA(BP?6% z&hSuc&s8~@z3M)VO;=;|PCws22_KNj`(wQAaD6%Di6c#^Hbh>W=&PyJkhM@iDwD6Ipz*MxqrV3(*`hN>6f5KgT zG6)wxtQNc@vQ>9@xmJkMR>QOZm;Ql>f*jsJ>CCSOt&6K&yuG*5cw1&351llB0*#^`AqYKaR;-76w$=>|nGF3A~^`%7Lq?S~@cye8;n1Za(-{*)|z+uuvuj zR)}`%s4XiGDS99h^xDdj2Uc(ZWcqzbHCYQiBMJFOw$*!)mb8O!VSN`jb8OB7A0@AN zRlx0lUiXR#!&!GlnqCX41dW8t&!K~bK0?y#h}LcrOsw3iB4OuE6U7qAGbL=-^nNQsH{7F0#2mWMOtG1(Wdg0c!OwHRL7o zHqRpPxyH#NttX)eaz}G1hmGGK1q69>nR>XB-P!I39F}56D9-DIS46Djtiz*qnFjg? ziRjzbD3TjYPvY<$U!~YX;mA`Y_*)Nm3qQZM`O)%;fUn`p@z#WlL&P-orGUQAq}cl_ zUGRBg2gV-P?!Z2i-AcD^`TIZ4{NMJ297ROn%}Z~*)Yp2x%_=p5K$eFP)8Pg4`*jZ7 zXz0z;7Yvxm0E>Yb(jDXnR0IVGxDM2_tVSZ#Y`OGQENKOZ4KCsJ@EJ%RD84ST_ex-kBWg&NZ+N7tVjjl zX_BG_5aOK7n&V&6^5hAQj*$LkW0PVHTI&@xH9K*6cxS4TF)GlibM`TLn(V)TbLn5Sf;#gHja=fp$VK^>Z#C?pf_bQL4IE{a3O<3OD&hDIP zzK93}m1D_KJNWuw4z4;~jzD#9GWW1O$J3sW7N#Wz-a;pKiac&GCOSj#dAAWJ#~9$D zqUif2)%{$%4HK0OH~xuUFCBSCYYvEgAa6gMlsg=jW;FNuo@KAs++^86&|o{$Y><{w zk8D5E3$1C@WKRMHc|$Sq7PZDJXE&3MAbf< zJ

kHk(>-ZEBYL_R3hihU##qA};urjfY41(WOI#@3(r1))#Z+71Gcy8Jc;Qv(RKu zssVazdz^aSGvn#ZuphnQXnIl=zu0Vtoe zUd88pILD%uxP0-z_HafGt-G%N)`TmcUtIXZjP4SxCR3d&0l&jr;Yy>`lL2NhmSuN4 z+Am5&mS~wd|Lh-RT5?+^uz&MaOy~f=9V~rtUR(at+PeEA)}5JhKm+Ye?hWSK4@<jx*i@SeT?Sj4rgZSNFl zW^!9;5?fUI8n*p&Tf}T(p87a5nWaXC4Sq2UdeK8=fY_AxBksvP7D*`1`S!?^-Nu)b ztUaEXK%(+gwsLvx5!42(sM5T{F_y-r@5|fO`J{^#5#;1wm654`#5Q%<$v)0d{f?O$ ze^lYo)k#FJzoUM-#Y*%(o7bd<_4{%{C4m7aqK?D*ugI>ng{Y-cF_$K2WvAp5fezfS z`auUItBAk0BOl^Z4bVmILYr(;m zn&3c-U;$N1Q8(f*W@0{8POyB;7nwm;FTU8~Nzp7*z3EHQ>u$XqKm+6>Ob0{(zOS^ER2%{%~RDRpItnf`u(M_{&V9rv_3r^}>FJ34YLUvdb4iXnRK)kcpu?$*N z1b5s+m#a?i=-(prEp7xj#TNZU{T9mf;+xV$_+c&!ptKBaWU%4K#w>GRfWC=j#NfNN zCjs)=CH}#NNC+f7+=n^Rkln-3f;x8lVPTfT+aP9*StJ3Hci*^R`9`Yy!yxeKdy6bcmlF?D*L1CWdKW(zmU>9DG|)Nr6Ouqtm|Iif^t)8%9M>)}r>-}# zx8f%ia30EbV0asA3s0Vf1PBd&?oVsG zYIprNPawB754pdI@-(2-7wI=}1O4C1x$RWPLaY;jZe)WFc;ELX#E z_tNs{SB5M!3hHr1PV9VdGCl4V%&)F%9j%lq!cjw5E zYt7v1tVWptFU9e2TEng+W<#k;acaR3J#qgfrm>rT&R?wt1uuq-<`GQi0z8NUvi7SipsV4A znF)md5f~)@!DoC88H=(TFsF-Op+4t`*eUVAE*UvrTz2f1?ms_tdq||24|}RQMqTB3 z`e-PP=llyc;fTS+Q#B;#5_;9%`BHZ)6_I%}&`>MZ46o|jWcP#stfs}XOdbS|#GEHh z?+XOW>in0iyUJPRnXIH=3S3{Jn*0zZJ0h|;NP@63jK0JTd%`tYzZfY~5X4(EaZ<|3 z_6i{opPw{au)?GiEUm_%L!g7tkRAQZrPlupcQ@TR3m5x%3pWmqd~EYg%_@3+T}1C{ zx_^M^jpZ=hij5G`*OY}z*`;nPg1a5RZR_Ntt3ubZIlsQ}qpSQi*Y{kuEqK}^k}-RG z#V6^*=5-mS1pi21q*dW28rE(iD zx9UY()ATMJ!o&}X&T#w|vX{Tclk>t|ali9+-YMK6Dn>cH34N+o>f(5nMmS_?sveMu zwrOVR7dw?6W}#-J+;6e4n+*;}79kPFD#N5-h?Ez-vwi_wP!hj~K%zRtbL3MX7pNlQ z!IXXSyh#`X3!5iDS7Hti1G^;NM^eZ-6>V6YY?GX9jiI^|TWMsvsnn=EUr+t9`0?mE zTz9EE(5_3-v5YGhp)RRHtWI3{NK3)C$8L~vWJGb*?OuM z7>_uwe*^9X==%V}&t*i4%vCXIH8n%~eICaaS4jz486*l!n-+C-7if^Y6?79@)VmVz z=c5zsdvuEhn5tKebh9-B|?^ z7)U7YArhEAg+jh$dbtxc#@U+A)p91+jx95)`s2dKtcIdjG`3h~?~xtM>1$C5F8CiX zb+5E>Cbg)HP7k4;>^l@1EYgddjqq*KqNw+@!+o;@_sm_Fe6+D$@NEuK z&emdakok4@-iVTUnuBTH)W^I#o3pO)7)&=)*P1nBA#o($LIJeCu=nrn!Sy*Bd+-%X zNf#XoOIlCzm7lfIeRrAoAoLRH#(2Uo!#CQq24U1K;_Gc&WHFX^jODTy3YF<-8%J~V zrDaGg`Q};mc*G6G+Q(Lm*1^r-NQ@w=WVRe3m&ZkIWOb@*Fr4EZqDUSLv1@S5nJL}F zw_mf;53~;yNRwm6GIE9SSSMXVnCnqE;_APBivH}6S3!}4uW*WM7y@VFJgJ39jP=>5 zh0%7GZ94GVrwUqWDT(DNvyKYm;wfQOG&vC<(lvh(`DW_QDZf6 z*o0;wL^pL}RugO(A6cu0)=5Hm(Xn=7RJ89J)+I+x3DnJ_!flY&%%gIcEgD8twAxtl zIgaE;u3Waz_l~Ipcv)Z08)57nW987}E<5sMQ+g9rcxhwEjNRlA7VedL**_z4&(-0@ zTzu;^V;nV8t3;Y7Pe#Z@`dTz;B0B+FT|*Nv`h*?Lv(TU~WDVf!_TN;eG4ZeAqpW#` zj^t7L;z&GITK1a&xP=J-{G)oa19pF*jS*!a2VprW@(L1>N8wW{kFK1e8xXk0$qOn` zlDa#?D_@~A-Ddzj0IWlZA^3eQJb8uF{4QOITcXl9%o_X=MNWdst*i(LF z&62#|w{hgifcKf32I2lTvlAdMO?&{s{PM$%Gof*+94&v!mq_Qo02biOKtR5e@U+zm zSkzo~xe1CRf`I&a_L6P>Pb5M<6)@0Z81$Kb$BGU}BINp>s$N`GN4CLruInkgm7Uq~ zhzGL#oyi9|#OU6yc&KcwS2D^~0$5;8r7Pqc3AQj5d{dNPCjx^2kP%Z1CfDyB4`bxb zs-2)jH1tbl>;5Q-C7Plmpwdds$hY~XCnDDPIRMsK-BfT|=5Ugkja;qk})Ryhp||Jl|jZ3~IGz!kqL(6m{CV%u$#Y1YFhFZ3;Hn`FS6N2+ZtcPN*cGvRy71gledWQ{0Qd8OUvvR{pBiLrqS4l)In>pwLa!Uo z+&SufY*cu>1iYwvX9H?{!~T=?1;O$pwm@MfWIU>yy!Wr3#`ps8PQ*U2L~;*|>~LRm zSzKV=lG>lL8rh0_!6wU9eD4OCL~l#T+C? zF6XJppb_=?h4EGo>n$i56~uasipNBLE6&L>DD0lb=gC7~WOJAfzRKBM?aEbJW3iB} zN7$5<&lM0v(%S^^O{`)*ZX{#hDVpzCpZ>mT=Ye^z6jY4NAKW}Wx;Y7=w+h=}tM&I% zoGrjd?t->-N0Kc^n4gGIPM>R@;4dSq=;V zuZ|Rug36dXKP%Wm487W|A2<>ZB?*#k9^3!ii3>cAHT)2rn^zO`0&Y(L*Zpl^^&?z< z0KxrDsp#M|Ncvvskw)(syGQ5yB0K`IAE5yvU{t0T&(|d8p_yq=B zyE+9B{_=o_&bE~4h##mGastnKk6iHH0O%KX21q;&O z0vb<}`xNhp%Wq<({N5EIyP&nXzzsX#o(AIqo-dH?35 z@8Lu{Nd(Hx%MjuqIh826$An1s_yR0wU+~HHf8v<-`W(PptMmLlb58DD&4CG6j_hq# z{6j=0k4c1YET+Rnq{7to>ftoBvz4j8y9}HqMgrC2k_l>aVgFiB} z8+3l_9}@C_ksJsf`Im{&`~)rW;&FUpMCg31uT3ZV`VFX@3>$3-0-(VSp{V9JRUNLS zR`Xt#-mRJ^;SEGAtGoo--!@yycDuTXFLXths1)JVtxs}0+4s1L2QCXY3dlB{^Rj*K zghDRif_&J!JoaT_$T+Kdiwr?aQ;vTx~LNT;&En7N5l(3&E-eZe5`X(lX%b0(7^#d z_9&9yxVsvY&o29CO6#WDK7Eq%eaCfgc7|oAlDd|Z*e<8^@p)dlaNBl@;E-~VV?t#& zIZP(W~igSZfw1aN1h>eoCBgzvB-uw zc0ao{+|UQZAx1{oj;ao2oa1;0e?o0GEv{vQ`_I_n7e{R^+hv=Vl?}=Nu)5E0fW18d zQgic?P-maWBFtV{ln9;(UY-3U*VD^dPB5^VQHTn`z21xiRYvQKvfIAkekTPKN&Xj) za%}w;$xvy99~p6e;*ut#{O~Bb=&vsT=mg*=DN!fN9wT9z8@kKc& zLp-L6&%{|`hp1rp5ukE-!MV{FWd)bLlLl0JY*wl3c$!Sv&t)Z8s1mM!RoH~Vl4UT z;t?lONuxS!`4*w%ij`msmwmda>z~egBF*-v*5#K4CfT>BU1G({XsFyo@c&MIKo@-q zPjGuwWkGafIk~xe?G#jWc`9e-uqI25JIK840tN%w4KMNd1=)D6@~6fv1!Z9|3~E$r zm+mHU7WL_A*xELdvWxp5H~=R7g~hf9LSQu;o#zL_Lh`h?ra6j5bEK$uwk3uu`wN_; zvO!s$pz+%sBWj{&*F8B9WGfe0m53P8l!|chN~rVSOm**-JOD){LddV=-tDT)f5E->=uRO^6OrZ}}q$t}- zN+UJCE;E?O_9fW${_eH%2wH=w>bwbfnBV*c zfmm!~Wpe;uiaNrOQV&@%ZYP4@%Bw`U6OG$`#-uo6Ooflkh@b9jO3fP*62PW`JJ`pB zk0g`*VijFP$jN1g!KF__#II7-aFPA(g9f-gwPoCy3Kr*_`<7zy#F#>K>Uvc!C`t4` z1VVf8{VIPD`CVF`2CP2yg zVd#4=&|kn4=-^UuPs9totST6?0IB%dQ9wx(>PFgb6Uhu{w0b7IiTfJ5~M`A&0yLpi}*tw<#YE?ZcxUXbO-U3ft&qY(}VYu zQaq=*2)jY|7yg&xP!qGkP{QvM?JQJA&9_7&$D%0kMvs*{u-{xnjWJ4vxBpG`zjq>( z*mA04FdMFZDw0(J@jwTQ>dsVtfLjnbG@6=S$kO-RrX&`TcV7C)2N3*_iR2?QXGJaTI=#|0%)oM}!L{xAZTt3-k0tFm^SYIS1d( z3jp?U)gA8496?V!(+iHJ51&Xe^dE>t`jQg_Nu-zau$hoef@o-Iwlzx71CgO0`?49u zx}>wp4eLF>|0mZ~DNq+!_Q z1mm_mwvYdVYA(Qr)mGRF*%h15I#_ryCG6p+Y+u(H8E0`il<$QD5*3I?1wh^dqdr<@ zQdWCLgUzH_(1AM~RsoaawcE;}ehMf${~w^!MM0iFNq{Yy>(h*SKjGMyRbwJX{t*(IX{VZC9vVjs<`wF?%va(7(%%MY|xXZ zP>v!IUylsTm%R+=H?bD9v8?aLMQv?uGYS$N+>D4U5Cgkalt-`7Ppa3 z$;-bof1p-w7__pwN@2i;DYG{2Xpo~|b@nuBjI^q-KtHY)OcB$l!V9yUQ0;kI8YCMc zgk=WrO1R-6oJf7}2S9?yc*#Lhh2;dl5f!p${_JVb&teTBRFdGQbWzvW*M_~qOB=Io z7V#L_Q{;EzB!GZgBzKp;aFAeti4OV%A;{4 z;fe#RyZd{bzc&%NdDfd>3{CPCyuRs+_zWfDgh5KOI3yclHF_@RZF+~@Ep}a=js?RX zEvDLHw~X1Dsn%#dH;Y!MULzb5E9Xmib|Q&GB*jmIz9M{r-#s0=#TyL`{-|FaEJD&4 zb))t6nL&h$wVPzQoQu#iShwCMZpyOg*BIu)m`~MNl&X}WEOTEU8O{kCAPbNRrrQ(1 zUZW=UFn9$?dU)yQJF4^ZTiaA2vzSeaK6)%k$X*fO)+S-M8)78Wp-&V6{h&S zUB!PgPBC>!5Vn??LpZ`TZt*`Y?f%-4Di;$VcR%R)j(1u!1NLuiz?P@vJezS&3sYg*V6x8 zI%c8STal(pXbsJK?(vgo&3<7U^fPabnO@ytVoU*esl=Y2M_%z#rIKN~S-W|gXx!@L zeic}=itzlaHo6|VkiYy1p`Zr{dvfp+m9`YJhLlkoXn^B25>C@}iK}+Eo{JGON~km! zj95$PQt$@iEJ)owfaOkFW`!Cq< zT|T0|GIET;M9+6`jg$*c3RLie7@HgVX+lUi!Fg}%7K^FgDo$M^OpAX@8VEE^7D9yD z0A%c%x8f9LflzNMDwEG`VN*kc@#J3$59mCEmFGQTmVre;Mh%4clplpJZHJ38QIzb1 zN1HFt_u&yS3g(SOM70N=2%NV|aNR@T{n;|}%Wu;6*AcRZ7D8J|;WaMSKd*%fr&8g) zy|9G=-2q#YAjW=N!{=#hVz;e34LToAG~L%*hkxXJ*84z#{vYx`izFg}I`r1IX_QN^ zIWTW#KMpHZrfU0Bc4=FC~&=0nQl@@@V6^D)st|R#ns0SM2a4tQZ>e^E}>T^n6coVwi<0d){l8 zdr}> zn90kq@tj7+^j+~Er^RLktZOCku+$0VLD+jJLtBl%}h#82~O0N{^Ev zq!F*aa_GW7=gQ-jTV{uQlCN#*D|&~dk6?1d*-DAvd62&j)`ojnB9QOeiiK8-e^ zsg4!%bzslzN$kG2=z&^n7x4Bz#{)nBpJ^mqy;_Z?I$pxK7#?ONuVXjO8ar!cN&fvf zA014N1ufo_jOxp5ic5JH?5A9RN{u95{P~8WUd=E48+lR}3iFdsk+HEX=J&To9o9)| zYZr^PgPr`<;4m%Q+R>mS$m<`&e7)w1bs_}z5usTx(oyZS{QKm$v#6P{{|#PX`aq<+ z;xhO_@cBaNYY$mlablSA#6>ZUdMaGWgrS^|u3#EC4chpwL~G-u9Z_B!`m%@}um_I$X*Oz!Kl7TzRD8uL*Dsn{DRNLKMmtxobOg9eRj zZSAl7)0RfT4aj(=-Y)m+fxg*GSko3UyMUX=?f^__vQqqtKTt5 zolespU<>~$XT?Yfk>T%b!J??m2Wr6Q0_Bq{HzozbI8IG#R}Ljsd)}4AVFjd;A{}!o zs#jUlIxo0~1APOz2Wkyeb||lx%wqHHKtlB<@~gM&+x_{6*<#D3j4mQ4~I8v5be0$*X700xdqRn z6VgUr&@I~`vC{4Q`a((*0@}>Tap_LDsPc^4}c3LfKEoaLt4)=6~jHQn(Un?=j&!cP-9UI;MY&os$ULAkqPE2}VokG)d?yKEm8bhQP0+Q%aRq<+fYnQ6$QVZk3Q$YPe z%^y(I~GaBz@;0dG0-nTQ)CY3JKBsN%J{q2h@hdp}!q$y(2Yvgr{{Ma* z58lp#;$ErIu+Q^oCMF>2=!^vMT=^cYP7{>GGxy&{ALH44VtF?rUQz?p0=gQJ+4T`S zt~{(>&_q4cMk0(5nhriE(5&XZIQ^O@z`Idy?fhN$s#Szo3=qBYtIRmDH$W*8wFVcl zmZHWZBL68LAD2HjrooXdKsAm{d!sPMG6_Sck%}N zw#?C*Ml==Db7s-1r`9K1;m?Ms_JQ!BhHwEv$fA90?HLtT^Y!p1R+|0d!{clx!Rac? zq~1hUiTSI2gm)P$Th<{_V$oDEpiIDxo+OR)9_#}wtDyvPG`?`9hUyIly`cb%ZAE>t zT^)iB_IY_n3L&6z<;u#6B5W>hiO#O~pbJ)@bjzOy1Ow1<;DPEwFPpDaBbiS}WLBz` z3NxaNnS?J-MyZEuO`KiVzldZP74@1tJw~_;VWiw~|8YD1iYhJDMiLA^&2h6jp(C|? z?8&(r^4V@O5+OS)i~NOygb4*a-k_uKkNP4>iM36E`Cq^e$s^Exy`qsEnKObT_66@G zro!YK12m2UfcDB&Z3_JOAsf2=a;KWXBY>YXb@&?s8Q=)_5)QthxZXD3;b_5)<-b#T_b{vUsA8#* zI-d#?+RhQg5*e6PMlTUhAB$r=^IXG$vg;i~GpVEbL{&=Bx4v7pCF!Yh$HMC_)lSO< z7Nd`zNh}kcq4+_!@&d=bAKB;KzDMKmb`M^3-PAryjo&J1>OA7pvLJ(5vy~X@l@g(2+IQX%b775|m zt}4Z%GlE5_Y6tU^zZ4%x>c^_e9o5}smnf{)ZopvF3vNc|VJJqpHYmr7du(<1YEW=h z$2s)4#>{g|ud36l&!TbmYX93xKmxzZyr*@)4W7e_TRBMy`Sd56+^x?jU7Sg92JlI2IDwulNzRJb+(Lk~P zUGlUR=%D8zYKrno&{U1}DcW7nT{YuJZN#OhYraI(u? z?l2{fXM43%ZZT!Y9XMK`q(M)>)_Y{JuJ;; z4`uxJ@Mxesjgw_}P|!_6hk1Yg>$_*+%{JB!Rq_4DWvc_vbV%z?VoUnv5=@4Z`8@rf z+qB$P#aA|Xad{=vE)4LUR37h)Oe`x`OB3D~y~+x!O?CUZLFBCUrsyOu{k3ZdxXv;K z-%0-Msrl&SCHD_negx;VW&LJ_<*_!3;OM$#{WXe&(!Itzuh=J0EACwHF3X`Y6>~!r z;bZIBE`zSbDV~wlD?DbyooUD6fnG!Il^=fD-p#ke4QkBIL+h_foJ0AR`>IV>3u4Q? z=D5s;vw|wwga?lQ_Wga9Eyg9dkZcXxLP4#C|Wg6qbe;O_3h zHtz0h+}(NmeCM3+fA`)eRb5@Xs;g^v&$ZUm4C=9@_7y4V-!0e-CyR!*oI5Oa z&K6xhe9r-$8XdM|lu_^7j%D(5;CTOVy&^rs<%+WQ{q1nA>$0C}+J;$%V4l)itI3|v zX3DSz*`WzeN(%KDqR>-{DVKa=Z)T_ZdQ6Dw^SIWM5Ei5=yga z%t~i;Od5SR=`>%Pee%8vBL{Z(!8N~l@X!9XKE_LaE&oDn+j7bVW+m2hdV{?>Z#%bd zSwdOH_c5FOHV=8%n2satqj?eGbhJ3^BK3F-mI~PAS`jsQQ zQQ1=*MKbq;)?N1DTTZ4&bP_@mAm@wK^#I^Z6;J6AmL}7P1$nk5lB&BJ26(IVoj>Iq zz>Bdrnp`P0gD$78>k~Ya>Itv$R}o)P^kA~HgSyEQmf*3<7dSSMmuXwRE;l7;qHFzfsBa-vuEtT6jiOQZzr#YlB)a_3b*+Sj`&SAAr<(%!p@Q z`+6;JX;!-D!rpFou&ys=+3tlWbIoxt*Xy3ViXB`>4fSk(2N1>g;??dmd-#0g>G@+c zq`HsweT)ocpE99Q$~|J;I5K;iqIxWc<4Lnfsu-Q+ziH~rWDs&0nT?-tI^-^SNShsV zs$U^7Gdms4R~#dlZ5#;RAJ13S5rD3w=^MZf=X%utGGGs%^g=UvWVNlgMMzCVmXn-G2H|ynk5O3EcTXyYYUJ z=_=a9c9RqXW(zBNJw!^5(|A2tVx+2y#v#2OdoM%C_O=<*tH*tPxsK*+6G3@*!m;jr zIihyo4I2dYUXLdfjW%^Jh(Pd9ZT*92C}xB_KR@R_@6_&xxXQ&|b(E^!&}Ze)u^aAJ z9mmdKu+Kx+y{B8-6;9AQp0CK9(!M8_;Xkzj_bXv!rhqBM^iyaPvuns0y2kJY=DF(BTPBzEy4x31aV>b#>326b*X^b8{)XQ*Y*)A8^Gq6;CV2d?YZRl3 zT{n3-8j9~Dan`zTvyc+*Bk#1WC2bTbPPj?p2l+>mxF2lqx|oLciS;M(x-#hZceYO> zb)z!TMj!C3$C7NPR_}9azQp*zkFx}doVUMM&a?nuznPY$kxC~IiJy1Ue|pi#WUZ90 zYR&_8t7mc6ZOPO$cG>Q`n(=6_|G zz;)vo+bJta?VoCL)h}Z_VDaNIyW?;7+~t4Ga^4Cf6t#7dE?yRG(EZfAa^5bSp&TKwqPA7VmVE-e})e6)Mc~CzZv~j{)<1F5rWNy%boxl3N95pg?dbH(s6GL ztU!gl31-(T0JC;w&2BtzLQiz8gXIgc=Wql%AJ4S0&5`kDQ`O%@P&x@%jH}xpPOd8f zLaiRhiOiuMZ@%Ua-lIny-Rn5AV%f58H z7AtCZC72=cu05mw1N`(a{>))@fSniGyS!19Jlf*NwW6)=zIfHjxe8VQxqg4>5iWgw zwrVEnpAVDQAfGwAi5*YmDEp&B{U8HZmbIqUf@u|k{8lZ>!GLcIHc5V2)sEz!Rr9Q>&ND4+hZ{z|CG)G&+?UTubGGmG$J`YTU}c~X06SQ?5nK@DJMkmwdbXlvc6yNuV@^N66GPBw13W5CBys&mOt1q-SpiZv(r3cIWXN@{DFPI+P zIN|Z#C_*lW2F#C5z^lFnyVJ)=5O(&9X)UAVas{Y5U1KDE*e*}b`M&B{#u^{YtT~(D z_Q}>s=s43gGWONul%71>K8+l#8`(Il>-w&o<^CcW^Nk%<0V>r%f&gln!F%h_tkhWJ zUErX5JbRp9Uh8~Kuk~a_*M=pqf$O0R`k`}Ib|%FRhySMUxkvjP$WsS%hh4|s#kZ38 zDNuZjFX7ENO;hPaG@k?-lv3(>q2a|rcx{3Y<|%M;?M~v36!XIWqI*iCf@$QU3Fcq$)_?< z&}dMTMTy|3%f$a`v(h+Ley2fDvK^WRL|MxboiX*8(;ClWUQrY?tZ`_Yht^BfZZZ!G zmS;M6=Sw7`P4lGSAI)U!_P096)c3Yf(|&(eaJW+#-dL3cjfE&UsFf?KGAxFf_t-B` z#00#1ucX;FqkYyTDW{pQZii(5HO+D`W@>iV9oB+3{@b$udZJM zAQ+Opo$3|@)BI^^cKmE{@;-HzwvX9+;P+#ie5o8$4Hvt9H@dm!2cA`|nCzrAI2^9J zAI??=dP``GjNM%E7k(p*!~beU9DS z#SJ-afQUhCQZL4Qrq;p{w3aQve&5RDv|D|nCZ?sOL}_b)g>8V#Vl=ISj!~FfQQdvn zdUH^jt|c7_+pxLRAsvi5Q0F>vmwDBcCJ@*Yje+G5d{Y^DDzVOEbAv_geoY97LG!lk zar7Cb=Rf|Q46cy~_(NT#Aztq!&6tf3nOR0Ebl6x+0{AN0RdO?4vYB#LTaNu1Bind^ zpE|qBu)Iu^6Z-782NVIRMd<#qlVXT7fX}7W{QEDm+!X^(rB`~lgwFF$tES`UGj1u#M}w2A zCCu!V+r@Vvh#J35_)xK2hSCcEVrSyRh){v~;LC@+mYj?>0UwK*T-|C%gcSNwZHBJ; z<5k6w;zvG;)J@(8u)?79X~=Cs;?*iYen3t2AVP3WUSnFFJ$12KIlPiVaE)>4vf-x+ z{pg;t#&J>wPdwC7;~tuK`&?6#ZZKwP|Mi~sI%&13O-^qk;qbS3bE_VE13V=EJ$#!r zExWV)&Sybj5~UHP?bEHdwyA7pCA;>B{G2dFeE@<1g}jW#PdE5_#RV%w9+VeNJ{*?G zOK-@JNNKV3pa%9OIse9pfgj9b-~=WE=Z}*xv=4lP>*`mG{u&4 z#fMm%vdWxDIk7Azg`b-+Zk<)zHT7XQ?f71Hjl`9ky zhf}^by0Oz}X<+FycoH&zsCVOf{^@Bw>eT|gk6s%@+*i-juz!4tpf{;TBrb+UY$FhV zEsm&Fq8yTuOlOPI6)_(ag#I95ee3%{TEmO37;Cx*Aq)CA5thI~wrBeDM6IM0woN#< zSYu1nF^K@wUk55!!~Sjl>#oc!)3a+4q-?@~ob8Xbah=X3>Y;Af;U5d77YsuQq4ua` z31rdVss@Aw^=?xZwd!mt=3v)V4L1v6#a!h1GnFuDt9*khrYH1hP|AKm^dKN@T-Yq@ zV1H1h(HC4-Juf@&TAP^i8l>C*U1dc4@w3oo*#w^}R&+CWe+uFTbfnc4CMC(I#N_j- z{6XBY?;eqn?ojR7JshAsrk_hKCg~?DE!s*&Qly`b+3j$@>75sj>M~5wcwnhPSZApA1$w(T~US%CDjzDL7t!#Ya&vb9w=Sc#2PIV9&1 zu`cUa4oaam&0%wrh_bkS1+m2LBv3VlCDKUfN{szpbj3@`aO;kWamD=HnSxy?{`Po5 zdPNQo_k*svewSV>BOQ=|izUilG4x`CVRKqZ(7i1^OQLms#bGN?RH@n6UcB>*D>qj3 z_e48s*1LH}msVL3oQNa7zPgh)2BgljomeHG59|y1D|UyzFvSp;U7mDR&OVyR=S$=r zswZ+m%7~J&BY6&EM@j6zY(fb%9plTi7i{nilNl7lI1;ojAQ)(aojUZkVLf-*`(O+s zWd!-hqA-0lu;?`#XKk2V-*i0jsL$Ynjq(-!3O)Xah&gcxU?ug^|XMWkp?Y2}8u0lU^(IL-Kx@NG_F znnw?1yQ7coqv@Q|o1$Mc{K)kzwHs{7Q)|ss5esvMRXXI|J_NU-IEG<8qEJ>hhs4Kr-dfY~ zyokHnGDO9nKv$HgE&~a(y_i1tL@I49F>3lKV)1CLHbS#xpSvFjJGGm?>zGI!_ZLdl z8q)F~hiAy~OfbG@L7ert5{nIGAJ3OntB8+~nmW!|qmzvIHrCfKpd1B?WrdYoQ>P1CGJRee8%o*QC1jOCw|lbMJ|?-g(sC@g+{0pXOdD8_&*B63#GJ+!sq5cBWvJ*bUp7l zy1hAfrEdc{1{6bkNrij-uBP(EYc8m4+uvFjJXF}!=t21^J|Ah}zx>wEBbP;R=n`um z_z@g&U`j+M%WlsS)D*^Ek{3_Evy)^^HxkhZ%qNt~SUfXzkRion1})a?@W~q(bum&6 zR8hkFnjn@dT!DtExmD;8;K}01sgj*2e!i&cQQgL$tkg(%s@u~KMq(2yY{QbL{PgY6 zR_A=b=Nwr^Iu&B3$gm80@5!lF35(Z53C8?I>WB&N5Yr4xPvsbedAhMb-hft}i?wA9+En&g{0kZm>HahD_(L8bqdF~y#{V0i z-h5%%s<5(e`tH{07tKMFE;*jwb%9WClyZ`))>7#C8#nZmHUW}LZct^ZPoLFX`JuDJ zAddvN$UBzT5N#J|p^BXQWx!dd(@7MKTkb zvx6va_r7+X*y!EsJsQLDly!<8)lV`|aTE2R80ZSU<1FMIvPtAmTSc3^D|U;^-RXv{Ix!3;XbL9w z{YN2FxaE9nLAh?z_%Sj=XP+@YAN=QE)g$lsJpzNSgXy92b!4)qZ<{??%Ged(T|bL> zR(Xyzg`K8Y_38B3?)UZAiyVVK1BSf;P z{KF@~q1Nu+iOb+ty_foUD~|CDC5d zyOn~XtZ`Salys;dcA^$$D{D{vWz>4_Nt1OLH-pFaPndI)KkR3+Rwwuh%9$cxoM)Xt?@Zo+v$B42l-SsMj z?O12jPAZ3*Ky4BUiJ$RCa>d?4Nut-}^-vf()b>fT?o|RZpUoyUa`1++7X#mS8#qF)!B#c&)n{zNd+H^VR0d$j`ox;&3C*_Jo=XJLkXD( zV9)PVWJP1T*N0xAUbt@68_#A|Rxv_bv!u}oA^$quGGGi? zjQ034@_48nK;s~e;E+!^9Kdx%n;54^c`xkz3@-HD;LCOHb1l7Z??qSjsS^&kDBiDy z@wGb%3LOdWwv(Ym(Xv@`sl7Wa!iWneQz__iLJ&r9K`{Mh2Ja~4L8A|E9Yt3LX?n{F z-94KiO^`pql+o_VQmthC9~7kec3tB@k59OWsBZcKkWiw(D5JjdaM5JgboH6=BoDw> zm9D@vMgDSK*?gM6b$0~COSa}DajRk(>Kp%Nn%8c1N?ogWD3GFpaTjkpL7{$#uGDE) zF{P}!we@|F6Xn#jj4@kXh_OFV-#`L%NG-952LG8eve4)C$`P0fiM-H5k%J7_0e=#* z02_1O$hVdYfTqY7ri4N{0LnUn0jqGpwJ1tHdk)n7mlh=NI16Soj?bV9NytoCR(e_aMc6Ov0Gux30! zggbqwG=Wr>YUm5psr@?UHFqMRggFfCua$7PT<_^3+4F-zZ6fJH0oyq>T}$-?>N=uT zdqWU7m((|nJ5RxN5^i%GEn9WmL2D@c19+RhFt$y+0B*D^&^d};K1E$&tCU)PqmWD^ zIm}O6T=~4BF8^>O6?Ep2NN7KoM5|3xFrUBt)ORn9;-_e8pkCKNeN*=gw*2Dn#>_p3zoEZMvwnv$k%&vTg z(}&WVde>d4W(2}=6+k!u@Wax74wL5BQ~Scilr6Z9h-YF8t5KH*B`Q28r8-(_uyDTK zWo28YT$VJepyx%cUR8^BPDDh&BR#qy-z?ada8alf-d#GCe$loYg?RjTdLR$cp9-B? zGeJRXi@|3bCe)0xeM|Z)?cLA}YlrU?hoeDGiezTsF+^BVL-*6$DW5lp)daRn1RERN zHTnZxc5`#9uQjaUPs`oOYf{=Dxt|>o+-!U_e-JOm{Ki%~%Rll%Mt%cph)TB+yLBNG zFwGhXu>|7CcN^4t04(F16d>Dp)zqk~oP#FA4j7p_XNVAlgv}<>R5$%$QpGfM^ zn9as&UHhI*9vwg@SNUpaewC?upc&oXa24>qmg-EB`>AN!*>e7Lj^R9FydheTxwBy5^bs!8rqDm%EgQ7QGu(U_CCA*C;OnF-o|Pq zMCWXeR4!+#jw9zVV}KT9zz6uSf;}r*lV9t(1wiY~vOf_!Dg(OA^!B2dau<*!+R9`=$)9fbHBGa!9E#&t} z_p85WE34#nRKu_KHvqOnfvR};fzoq9qsc&JbFeg~OLCW721Q>dK29EM?pg}cs$%^?4c5H`bL+Ht7gxYTsp@LzBiWZ7n0FbZ1nzdqi|>j7+arw$w?*P1=^=S#eWsK z#606sOX`6#?S93^fl8=|$==BG?Y9M9c&XuZs*3cv@!7;8ywD5ZuRNjO-F53m3stla2ACIo&KMOMLPkbUZ3qhKYEV{${ zE;?KigFO#o1ys+_x}L4utJl1wz!N%sOGL&pZ-wm|?3CIP5}h^pQ8V{DDBW0D}L(vr!-W3 zs3d{B<4tb`&;H*OY#xO_*{5$yh^Y6JKXmpZjDZgstYVXfkx7(Q<{B->sPwHUBFGOu zHxwW@G~9;D7%6_3oWg81IXnd z5Kili$A5NFZw2wG5_vM@o>_anX5%oQDt(5NnM!Dm;}v6zDOvb7TV_Trpb%PJy!UL8 zAJX}%fADq%a*nieK@}(+cxZk{{6LKoWAS@hxw!x zN3{&$Eyg>p*>b$+>#jUflt>Vqp?UPa)k;a_)2+ zRD@R0Xu#kvjCLdz%k|mDcG1yp7b;494N{kr#c453Rj-mnw%CYdI?h%^*$3(qjf;LL zC~$TVNsR9EF7K3$j~%(l>I3CW;qqV)%&jyEcD?zs%v0Gz5f{B=wvVlJ&3Dm?(86`Y z!SPI`TrKoxoe_#E==(smCTIFFS zjIb>~){mw}q{e^FAKylWzaN#T1_GI3pc&tUPf za@{4zG8iXw75ts~^nA8jZ4l$deIM*&t&X2xpET3g5PJu6cUdMzI-(&TVyL|0C@p43 z-W{=-QdD((BViaIimS>CWMFIfyk5CFsgFIJJcU7f?l}-WZ&jUBs${QE>3Dq@_ddW# zi|IUBtLJ;`$>-M!VZq@VvvZrJx^w^i1)wT54K*c+{u~r(ss$*>Uy99yRg~yIu`{9n zfkaOEm6&|4kVLO}cj8aYS_w#WzBx?)eeZXl^Px6N8fgEbfwtF;CNkBA%dgw*C=Yfy zUb#G!4K$hWgjG=MT{06UJZ97-B2&LOa{8^Cl{$aAcZ7{lNI)c%54q-m_U(6euRvwf z%A^iHMEZP!P!_BEtU!S)TJ$HGeOQ5KlCXFQ8^8z(+LDp+F<~V?L&k7AbquRdFWI#( z^nS>D>Lk3wDOf!dE{{Bq+D3%C2u9zjOOXqdec)`yS7T2M^?_61)>c zSz*bSkPsGnXp`C!h@J*L_3(g5sv`+-V&;$1wGI)&!1*tOnjawO{N*{|g-ama@XoG zl`xTVwVUGc*4Grq1avsn!S4+?#r&_TwF3Gu7^M6!EWoK7#jTiWs!!~RO24|wKxQTg z+wzF9X#L^bWUoCnc}fDG6cP$jVr8qaD4EHqT}PTPcXs>a1IXTb%5_?5F5n10S z>Atc5{7EfPUG;diu0y#hM2zKkcRDwsX0z$rs@leO)#iXO2=M_5cFio?ckiIF)Wa}@ z2Z3~-^VWfNFIgXV-|>E3m$OK3oUAO!wov>l4$;&%&@h_${O_<(C>Z(!8&PcuGlYm% zmkRBVGz)}f^W|~nz#sK_h^#3zno88GxsP2CumjZeT|Zox6eyA3&lBX!nRsBNlV~ew zF4pU}1+&$|A3j^!73*J26^k_-K?QUD_e((x8RTeP771invOw(B4;EWLnO{p|nC9f3 zB#11LDy`;Vw_Q~s3ff!A8a`-pDp8u}fIG%=?dA|4ED)C%rE0K}2xYSdVVjI0_0`JH z{Vt;fP^q#+=*0Yh;lpK6cw1~1WC&S+B289vO3ga*;7&@xOvxLV(6Yx!H8Q?#KD!0u zOr;1+Mh}vbXw$TJgk8NQOmI_^z0IJ~BL6XK>Eny9pEFk0z>B(WmW_Dw2@$w%mS2BjD91$nCR#2jp$y!(Cv zozbe{6(G0G^n+sF&wU-9MlA+K%EWT|&JSw!8ZFm#)*WTn_ebND$s+N{Pr03ID~K5c z53_!Y5V%#hU*AEGIINjzB=xJn)BV|C*%{8uo~NSwp#d}h{me>WV;OPdq|WgrABd60tju(qt_3e+KWMY!ZD>DAb7d2oq?Y?z__?j2UJ&#wq`ZGNTt4jQ>xa2jN zEFNO4X4ga#7y>w)ze+_gXX~_tY*in&!#+MNb)gfpSx>a9;u9L-vnJJS5 zv8aOPT8x0dCbdyx-^mU^vHF?N4)=d&B!y3a2CKxQT?qHzysv6Zw9^# z{@3NpyC8q zvDWGg4pt2)zF=77_qP1Vc&}l)m^Bi&b;L_z?{C_O;6@X4TetS+-N+LSAF8ZaDVVXU z(t0E&Xdviyx;y&z$PE{uV4y7{{`oU|E?6nJ!>HLt>``*b{) z?9J_hKql8P7rLZQXF}v3_I@53bJNV@C=CWb+)EM3ch!70aeS~ir!PHo02#|hFy5Q} z1ay~H3l)xTZla0t%GqWT(mE5poY(DS#7GwWHW6-6579)}Wf(7Svj~B;$Xl32vS}nn zzBtzXIrFcFq#qVDmz`tBj^8e@40^v`s)IOLR5Xta4R&NF`5h=|vzoedc`oOZod4ZW z&LMnNH+20RBKnbLrUNuFy%MPXef@QS^|;)OM5wnJb|=RFyn{YM^?-Wj0pc|F*k~@^ zWWLjF+~W&R@}<|CqAY6qR!w_@$yx%r-zu6myG>Hb7jj8M6C(T!1HRm2tE!P)-sj(7 zw51121bE60`bUO?!yl>KfZ#}+nt=)UYdnk|R-^#+(eOE?JU9C)qmkBzqoKPf-_nb+ zF<#`)S+3eK<3h!LO+Xp|w*$DASaGrLM(^@*eS*=y%kesUo^`8ZnZV>}momxBuhV7p zBa&JRgJT&ynS6l+8A3d%;y?g)IXnT^3-%qG# znGmPxc4unO@{>Egh38;XguwaDJOUQ$RSFheP3zqHpDU$+4uVLr>tfVPfTc>p5wMKH zsF9}89vg>fTT0vnwH$UJ55BD!7C(sdYBzq5%Ge$nooMs$#sXK-Oa?`$L*6{O&L`VQ z6kp)M#7nJG3#9ruwG(C2d!e3NaLj)SohVu#{_B1hcFPi(fbVCf1jg;et~Jxsa5gBA zx!LvARD}~@mpv#7db=d%ph$^9=q5C6A-;1yNen0dChVCw28d>rK zvNv0h0+P{yJLG;OCd8e1l)~ajm@kcPUntlO@F1){wC{R+A@P1OzaTHIvc)^=H;)iB zl)t?=ztX~#jQieVbl*y~w4UMac-q=9qOO$-brNo%7VBCl)&Vz<-a5=+QWGs=X2V%h zeskfF2OI96soi*(*AXN>OrGsc)h8Bt(fk3)g&+4O7rmCSd1G~^XhLG5vxC;vmR%aR zZa}*J*b8JtM*a93=CmG<0<5>$rZaNm|b?|qWvNc&96cO zF9ZD;e;0-aZnuLRCYKdb`<*|4jHNow^PZllYoQre8S@C8&vS`Aze_gZ4_k9yQgXkN z7pRyEDCaO7+yJQ??^|2ObQac+3qvPS0aiuo8+7&e|e~ zoX-oARRgu0*L#h`f;q1x#Dlnw%DF8cu;lm0Prb~0^&C<(hS1zzsV(;ht|=%rT=R7L0epyHLM2z}G=9hN z%0s$D4OmL=o~(IqJD1###yfpUU0;5ET7kj7*dpZqnqvb^{|yWN?&z|xIUyue6%r~~ zG;Kf2S)|^4HYXPml24CdzQ!vNN`(E*dRoNYcM}eO0YbitT@^oSy<;759^B+UkoUCM zL+h>RN21c={yDhd*VPNKs@D}jsY@m%`YQ*Dq}f*d;iv=x0h*smpcf*l?eAvLk->?YD1(R)e?_6Dk z$)sQcLRokq==EBQ9r9qPkg>h5AZGW!eu)&lZ4e12MJJ4N#C&0fFxag&uKLH_|Bb2n zU(bq(5Ind{rIY6miByVAI%Z)C9l;;D9Vdgz`pB!9+x5cHX>s<5g)M?mf?@KQ{K*L- zIjLs0TH^j=<^R6N|M>$vyefVoBP#<5WPk61|KHE>Vc~Ag! zt_PaCueeSBf4tG(mxw-t?M}kAD)t{7(f|6+|I5v&vyDoBJ-PU2fj)IV@V2|^Dvuoh^@&((TG zxeOl4{r6Y(=>l2mr8?6D@Jf={775GW7riW({QEx%!@o4!ZSfyJdOnR}w|L!#8xMaq z=k;WUGfi^4+!np8+durDp%NG+7J!u)Z7XX3{ADvTYiTbZq2|jpSpwii_lLGp~hkl)R-j=>}BdoPJig!ME z{DRpVBKtxvA71a6!RVQ^ORqBYhJ?q+GacO@{m#MP^_vHp!0f%iW{AmBtOfD(IPm+Th(&m6db zhsqy_Oj2)iL2%jb?|81zZu+hI>=I8Vi$keiSnlcAJn~+GO2~Z=BYTnY{;bpdYz+6$ zt*8L_)h=Isu8ybLzmkLdGhEJ3^(tk$%53GDsf&eGF zW$DUg@saKfyi|FAH_MOpY;@I=$O=C$cvxcM$)_ zM6m*YzJS;8c51EoKM(2l6Gp8Hbb?qJjM&g-NsfIHs6I3d6R7=%LzL%(Kj!aM?4L$mfk34<) z<_u`i>9EQ4V%(*+>KK(SD3I9djXoWcy?SnUzuHe4tG%4y2faM{?iQet{??evQ_^a1 z4Mj+0aSngEUC?(SKpz1lLjh;{zg90*TO{a$D#i%D&5xut2wqNsOFswj2rl5B({jBw zu|k<_Ite?^?LP=*T`EUu*SkR4xiS?1I5E3|V4eHxNesj-sPUEdRHb(%QJ%jd;?5zT zs|j#~+ps&5pkAs{Ssnr29sOk_5kn|_UuG55aVhC?jT+;}y#9DzZ5LgThKoB$HI>b% zl}5j&w^Do@)fO>!lF!pdr^|*?WAf7Fww=x7s=lPwXkV_ZRCH!9o=#WGu}(*UhtH^m(#Vsl6MQq-PcTBCp6 zWgf3T1giiFPA-4XNQ?V*AugZSB3B>{I6wOGaGAZi6y$Z_xnLWz>Zkqh`e*#?cYC_I!oSBG$55Z6-S9a*ZFBnnI2L zEs-Q+Dwj@IAeC63hEAoYLn+-DQXk>_AG|~gv$|jOT@Iy zD5XW}gYR1^coB52l`FR)U#WE2c&Q>F59iDj4%^!miG2#ys$VTc8XSCC)7Rtdb(zJEyuMelL(|VMGtJBpM z%BhT=g}S`f+r}kjWPU&LVE#lhLlSqEQw$pp=bQO00CKk0-xN!YZ-UJL77|HT}KB3z*hirX=obJKp3fmlZsTC;6ncvEk}UCR;fn~3&_*|C}h-lx+q9VJ$^(r$Yfi| z!=#N3VLX~LRavD}76N9}n5ByW1Agnx=U3_aPzk}XVVE6H+6yV`>FPK-;98q2smXG^ z!{d=j8M0xw93C&-#BVukKws}BaE=3jOoNmQNE9Xy>q8{mMe;JH3Bhypw+Ys99(F$sT`O(OY!yj+`GYYP8H~ z<6r03hj6lpmuQZg{B$%z&wo__{pI$Z-7fYOo5dhU_|ado?nxHMor0gK_%)nr;*)ZV z!`8NFi9$oTj#?Ix;b%94UFwG))j*V+Fa~0Pl}Zzh)Gedsyj#xiNzhW)go~SQcfRrA z6(K3iGOb3v0((CGXtl2%0r5tTW>VQV1Xv00<}+O7#MPD1JHr z_-X+pGO+JQ>TGFe`H!>pPmwpc-W!XK7>(mC%EwLWy~E=+-rDZJtGwu51h*S-7SfO` zROrwa0RX0N?T;s?3T)+yVLC)JP#-}+r>h13I_k#$2s*AFOurs zKR(~oDgbjMNzw}oddWdlL9j3gE8Jagr7xaa0jlE(+)c)cNp)J|Z?8>+I>{J+R6{W% zC_`g#=HufiuE6Yibx@Dz{?oLk`v|6M81~~Z_OKuLu^+hu% zaE$!a4CrMKzAf*2I+09bDS3^o&@PMKxd3M}=hu_8dAW%~XCLo<+N#!&*Yk~hut>St zZPDF?*eoX})oylLRtm&j+pME3=T4l<1&00XRJGLJFE_mfl53p45z=U5fBk^8@@*bP zsWb=x`D8Koedr4;2;mF)k1f9%X5%naen+2dt8=)CFc<{1MQ{b~HfcW_1ralm$KAnVwK{hV zpL(T{zZzLI1?kU)T*3gh)OS^2QH94(Y4GN~;rzl@oQsDBRV?53oKLA(L6B0hyxHru zKbh%aNB(4|JMU&tk$)Lv8kN3ytf6VCky80)lxdVsSI&Ht+`=PrwD2j7MXUU)09oMu z`dLvWWz?D2)!r~MsY)|Ft-|DqT2d{lMsB0cx{_@z+3;TJ3t8c)g1fqmppMJmpL>BD z!s_>KqL?qv`||?OSYLUL9<8YMzylCgAcVC27%pFHJg#8oMnAc7p%d7?RF;g6hu`TtI4_*-{%}vV<}^{~9f=s*XLcVW4$j*Tbrf(g^wo ztEoKEyG8)D`qJ^xRIW)&E=j7DOn3M5?{G#0Chx1S&rdyhUP}(GAFbR^78#eSr&15r zj5iyFGFeC|k`Z~7V^wseD>FC??6z_oXK(+F7i6KD&bhL?;3LfUrl!rJ6r_oGJ!?+q ztpVhq+@SlV{AUCU_t$0T)Fy+@HJEjLO=h#25sw0J4)0La>TwNaV+BM;M+|>D<-dv_ zk;HdF*zefzSI3d}JfBJqAZmN%875&fV=La1m$%%*?)$Y)w!?t07nJ)4`Ql# zu*1#4XAr1bJc(ig;X1{T^Fy!6&!=m7&jl+fN`+D(&bFVGDUWOh7KpGiN`TR(U$p|V znN$GYR7QQ!gI}`5Du=yE&o-3&L$EQWq?)%U`W*lZrnWKyA42RlZ0 ztXgfc84Me4ldZsE{XM}gIPm!+CSClQZYxj@gpjv^5{^kt=A4)dwN6AxJzG~*aD8t1CVZ7TNA3S>Ba{cpS=8t~?Fa!x+ z<@d&2Lu{t4b2Q*3A#+w>n?!s*F~5KKih0cb6TY$~|1JCaT?1U^crmXe;|ht!vauGI zi<3=fmjnkUrwCX1295`lCB`nSZB7EI$vGIw)-2e+x&U zg+n2@Qt*Q@MY?f&yYt-FW&GlI{Cdm8Gl9P*E-~T_HUwKAaqmsI-A@+8BDu_ZX->!8 z#0*Y{q0Z2ATBQxFb z^w@^nq3Dwl63CWA$Kz?-Xp`1dS$6Cu>kYDb10YN8V9 zU-5pDYInQD#nL-|smA}}l@Q>kgDi3uzvgkZIJX=+E1 zx$8xe5lM~Kl=Ljtwj2cK(v!d}Op|p4J5So%2YEpKjxhaaO)@d`op9p`HpX6lu%f^} zHX~yUm>>-X%w$2Qe|4!UUMLv~By?A3f+=$ODomYFLz}fzvhDR7&+aX>w!YyFPoQyz zLr;5kqBoCFhnr5pucRxIs4;^G@+D%+e||zo#h6tPPdX6(;FfW|pAq5xccW8v`(kN% z2ci))KsA%cS0&?l2I^z2r{05)n`IAx3eZVmH5=2Cbcrw;QD3c&;SXNXg26UQv60dk zV9p_qZjG#Y4x|&oK*c>U-+-@N(z-9aooQy$j>4OkJ8}}7_xT-9 z|5p|*Q~yl;;L(fEmxrES$^CzEF`%%|Iyv1n2U(DP4(wN*XK}JOL&!Bej35kF^ z8D_{#G+!5!gdVariZc)Gd+uNngD!n%bs*p50be6CCZO7UEuT@pEd6pO6E+k2kMuo% z0~3~X*6Ok-|INc}kpdMO$e_Zf^CdxZ9Jzpmuc)Vo`WdYC@=DqvZ+2(kIq8-{g zxaDfG#|3N2x>%|?j(5K+X6W*>B*ox_(1$j!&OCPTH1lNyX;Hk?4RhDW>Qtwe zchaTj)B@W8EbcAPDt=DBEmQj4HTGaeDN_2R-#!0gNV(0eRNsajqX3Uo_|=XefLWM0HHEm5N@Qk`lS1r!w&C0*QwW6D41&TMWevdrrm*?LFRr zi_ELA;D?pRt!@kd)6vy>a&5=$Z!CQL7K}Uo3);hb((*p3k^H&Ug7O!za)v9*z7g#m zxNwBTJC}0BneQgwAs|R>PSvfwJNG`AFT;DP5H^AWfwTr(eC+jnaGL-xgy;ii%camA zQv{%(AE;RRpt8z$U9Hqjje<@0&iOht@x)*ZXmQxkcInO}FEIrw^%B%Q_r6pKGQj=? ze~(-;UrpgYeeT+ZlJ`L7q8$H3t;=4Mm$zHbmD7~85G=;ibS&D{0I6*%m1N);>Cu3a zsdQc1v-5r1$KSm1^2Iqw6cXvi(bJ`n^zV#{(>AvuoEFOXO4jBMqK3aBnkV z50TZ9kKnO-&S$4JR)b#2I-l4dZII(qntBy>4z)FSB zZLZb7fK63wzdzZKxkiGLPgszG3{$BpN~3uEUC(reY-$M52{kre2vGGG#T;z9!orWv zm*qy2pYoS}F)t`lF*`{YHE9b%z8e_iu;l0X`Jr(Q>6i#ot_IOSbK1* z?Xi!+M4CvM_(UoYP;kiD-PO*uf?671bY5N0?o-o&;2;Kxk*yds70`n5XtCByeV9Z zSTcWgBn8q4`x!#A@nSK!svj%&rM`NdndK5#P1qUP6&ov<|1Hodzzh*WsQhO|$X-BQ+9>IGCG1@mEa%?*>lZB?8kHgO<24%*i9IrbqQzarg$a)?7 zKZ%no7-tj-2rxDxMo`PE^ks-qN^QRbU7Ge_SaA*i6ToqY(;|dj!V~^n!>fVzPpI!0 z9xAcHrG$YeGO*!)zCVc~QxL>#9XVwN6xkhQm#Gz5radtk<$uMJNx60Rym1$%J))nZ z$@21?FUV!LVf!`9&P=8pyzJ9ACW$UgN~=TJ+H7y&SBxWwZ6-Vr34fMtlsm$|WJOKy zpj(8yQ!XzDnV9S=wa~VUkf|pRbFDjLUv<0B(@}j13_R%62j(0&v{x!$Iepb^<*ojb zEhtv^ivF#2xlYlnZS>zhI~dd(3=@FpdU`4RfXy)gYiL_q5k~MbCaseKl1xr0PPRHK zTct(u6hWloU5_ReZ2CDX#FFo-cJdBK6)G==7Y3)@EmrKdqt}B&rUtp;R2uxKrX85` z$h0zn6bDAG7RbQIfN>RLkcLJ~pBf%r!FM=3%2kaG^FQdI`CcgHJt(BCnpwCa&R zPGmI9{v6NJT4d6Tu5WBkMG;A-^62jBke$bp<$5X-0+dml!i4Co-vCPKugdVYFH8OQ zXzeCS=c~aw^{BPsyE4g1_k0zRaUP`to$X9sg4t72o&K|M6KM_Pe>?*s6CHY{^uyR0%Nrj380{Q_#RqT z#Y;pKA@18aJ(|md$1o=5>f(4TP1$t1qEV4>h!|tKxs4_Lm{c!1?pIZPHbH9wA1UU& zVga^SmHOGOAF{*U4Glr6sCY}*Q%ndU=mUznc}b=@CPEXM{KC{h(s8Er0ltIAH$#uy z{F&$1krKOt4*4QQrfkG$=MFy%4BUDoOB;M{dy$|k8#Q!F$RlM?iO5)~A>2^hdo+*U zQ%TQaex(qPS2E;jGLFRL5o}e~C&la26`I{wRvLY@x?)MY;pVDQj^p&jtB-*$klQHu z;lKTUV50l9!!djw&q;s{x$}TY1U)M4a<1|a$Y(o9hU~N7yMzn2_}c7N zY;+`XaDFc4pgfO$KINh3jEB}ksCF5M$2Q+^bs5G*)1VmI;!B6pI#UP~?{uHmPgFSx zUq>V0^pNfha@-y^Q(7Z?-IPq>pQf@wEQvvtac^T3XPrw#sVCy>pJWtR;5z3n92m>Z z!$FpP<+Iru*VB`oEM0l`iuBa} zGqm-Tka?jqBJ88p(iz%aG$4_>YP+YEf^eOBYYUt)m+qajtd@%D?kdK9AwFJ_b>+J=TcC z8!qUKce10fNN(v}7;s4V0rguEzylKT=*<6#y(lo?^}rGRKwIM+SqSmV=2klfLf?N> zvIubp#*2F)RBrzEuYWx1-!9%?18(43#PfYA|CU_^hf@M~hBpXuKmESMzt{0=Zt|aN z=ofS#%~aYllJFg(@A-)z`}EHR04Z>1$24gI@&EDt2G?0-6)j`Q@4Nql48U8oqqyH^ z0sQ`7)P?I`EtsK6em7Wdg75=0;!?n&N2Fsc|F%E>V_o0Z>$7|9`tMfwP=0g$e**IR z%YpAvRjw1tNV*&s|Q4znGlM8T9h=3SlV6K39LO*PAfWrjHp)k7a{Bv)8$X^`^!utd^ zA4Qx!(b*-yR{iD)0rwoUz^b^P9c(OpJmZ3ObUQg?b07Y(w=-l=clII|<95A&2sG;u zNZ%mmMK{9){Zi~f1Y6&BHqEny73Tgh>Cpp@cf$;oKhX!BL1dVf+BR;k@dTtF`W+t} zwg4wfwjW49-$nt!?#xnkqE&aFwoU?Gn!6n+66DeF*VNa-$e*-}301P-F!lRP55gjT zndV<;yfcU7z5f!v?cxXCwU}T#wG|w?t|ifV984{kJ@aUf9+p1w`%{S7K|@aHb&GF4 zV2~V$)(IdoLzRliiC;Ru9k-jmjIVjQ>BaQ}H6h^VBriM>15>8nW}f>{SA-8g1$}+! z9&!)1XwEm2v7aX(c5@oVE^++_@w&q$Gs7CydRTE126mm)o&JfQd+gr42%U0ylcnh z;<+rDLHkK(GwrBgr7zyvlwL@#&J}=k^3lq>b&f7ZAsn1}IfC+&i2S7WXX}3nYyTka062=LM z>u}TlvPhuiX6A5Xe8^B17M);=uqXN^cKIc*-*3L^$RG+YvTR0v^c^YJ4?&1-1QMBf zw@%}k=Z7Xl|5ys@B%%N{yvS=f#77B=`D(tKY7Upb?uvAPyKC#fR-E+uzCGu!|M+%| z_MBLh8jMaUVXO{JEFKn6T^8IvDCYVFd{{p67xkK}b`pFojbeY)Pc@LDreP8T0oJ0g zn)S}38b3YWziG_F+ugMy0FN!7{P{ZuZgmUIUmyJUHPnEgSVE3dlAPaLkgyr!{NiXj zl-pr9)7p{yr_2R}rh&-JP@d}UR>PHwe!>&OJ_CDrZe4Vrr*ND86AgBfpv@etADSJI zkR;HvA}t6w7lyBPm(r?TS`-!pmnD-{j7cow;d*)QkaMo{Wjpg%d@w>`rl?7Z2Eb&x z9pQ{$L+$U9qi!I8S4iwO+b@6}u*x_k8=K@$VE9e~U;%U~q!5^eJH!?P-so9y$#Mvf z)TsH<4SG=Qp=!V!R2kSpt6$a&eJ3^pR0zCAf%9VBW={u*np*>Ov^5cN#%4C&5!~uz zP57hm3J^+!(6gh>&bo2C0j@r}PA=Bf_?MqKQm8UuU{#bBoA^;IR|lvc3RH+}{~|g< zzy=77A-A?dLx9xVmou^_+ar;1q<%2rM6b*xe`+9D^lfq&L7xfN8IrZC2o|)b#G*7{ z05r-T-m_dA7oSn*-FXPjQ_@+}5|p0{*v0^{nNAc&5HaLJ5_#(q)i~(G;F7Unl~{io zk$`Y`O}R}hM-@Tbgn=|2Nd@#*QII@v1e}e$oEo6=2%pC>{A*#eT|pJwcRT{S1rP0v zqVEKF2+r#X>iwaXV4`XI{CFG)NZQYR_wsvQ_-~5hj%mE8q0z*pkU-5W_14uK1_7G^u+8+gpIChdFxn^B^SDm(yCZ=59DaB7E zl+b0!%%w{@XK;dDkiPhL4%@CoRMH@Jw)4YWw;)X$Mp)lp$w*AuKiDil?r5_aBb8Tlm9O#OL|vl-a!EXLx(XejUQ0mUh8jntdjdtV=-rf`Fs#orM8 zs#|Yd3vfb-gs#Z1b8zhUnl+Oez*= z+&x(M@qEDJsYEOO>!KJ_2}Qa@zh` z44cWTM7yAMj&&PNMbcyx&ESu=j8WG?rlYCdj&Ef2KR$^?(MKxW3g&86Sj|X)K>2*8 zHpwY^TO>u!3l$_Z;}tIdn(0J5kbE)zDn_`1@bl@ln|MQb2g?9Y=R1{*EQOTajVp-@E6vPTr zNV8!X676PT4^>>WC&Miof}&C(VUXSV_B8F9!^MZA6=ns`6=`308^~{l1_pKI4CjT>dpYY4MmS`ZVoKj3q>$Br9+e^-Jl0ihvqA&Z zq`$Sma4oY;S)VKsM#83{`sjxl(Sb!H`p~AUsr|y~DA$QY_td({g~MXjWAI6YRv4wC zX_iJ>wj7nF20FRO%yRQMM@t}{V&r4Zhd|}(+juTp(KQ7+c#;m{@j(}c>(V|2bfm1rLPe*hy`L#zDVmheA#qCzkmP9iR;!T z&x6BBk(HFtchf+Ds;PcH(6S>S6nHLy&KvP1`%?RZBei^f?r4iPk8rthvHqDv4xwm_ zZ9G@|e3mTtSTq)OC`DLwf4V9Xjym6b?EAhPGa~iyzndba)?Y>)i00$$b^w#UY@{Xd zIbVs-q|62qnsop*%VibfCnm`Q1+t1rSoHbfjH(-sX0)2FhgH^!9Rb3SNw2$)1j9`$ z6sd7=S&e9FY`2Ew57CeGvN^qJB51~K92br?-5_1nWx7WnmKyTy=z;oBa-eJ~Qolc= zf~~6O>ooHcoNsrRkz1Oq{>f%5(YniP)zM1FJj_Q*4E!VTt<&6xz%dW0syzuzHlLo9 zj1_65TN|FZwuoF8V5YeCr+3K!<5p|(-5Tstb=pH^geSvPD+Zhb#E@`k9`!$zuq?D< zK~!V}S~L)S2)NW-xVz)crfR8yc^>8W5ItpqfDPf&R~KG|5Tv0c^^yMwL=bbLA0IWt zQ>zqL0+rbLOJik5GF5suqfRf}3ZrfdI>|1eeVNPCXrkdO+yf>KrfLOdclcfM7rMY# zz_w;ir@r;xC3_Q7>22rBG3xkFG2U)mCBB+nZ+gA)MZo7wQbJ2~6&Orl8Vw>XQF)ICl513*)g`2=D1WMy^cqwR@A?+clf@uihHi9^noV zqngD1uNkQKR$@ssJBRMZ>J0Yj?nds0RyrN!H=dEQLxn$;Mn4VPMxbLc>aZRK${ckK z&Aeh%3YyVfnAh=ga;~dm>S*H7d7~4Yx_d7~BQOaL_Vlkz2h9}U2Vg(eFs^_eeaPpl zXe)Z-tvlHxXl}?4D{6vNQ z>}Yz(=7QC<_M;WhG)j}Igy?kSMDtm`fVYmoFT=1zkA2hddzGBYp57=>CTC;T^+nt$ ze5nT>YmA!x!L}RGSZpY{C~2jcp-e`3L7r(I=b>%xIm521lj`O~6~Xv~r;pCCtVN|W zi^vK*N+6^`0@c3va~yXUQeb*83Vm9)x<*C)cy)K3hayU|^IgY=hR?Aqs^%z3#?0^} z{f}up@hDDFfneXVRe^o_uU5xz*Qk1+cl($7o%Hx;^f6qLqSz=gFIpXenDfUtI5xXV zAWm+*GS!ve0i3o_Y<CP|r0sg#+i^R^%5Ay4 zJ(S~_k)Hjba$*5j@(9UND`td~0CK%Ia>Q-n;#5c;&3sXoOjRh7zWj8D^qUU8igRjh zV0!FZZJ{IR!bq=nD&&KZE!J*+D)MIo@0Qo33(G~?H1uuAws`phCZK&7-?1Zaec|sm z_V0c;fOs_i7oFvFmKUcsHT2{ytPw~~B zY6CjAM82NKl8M|O7vc04IwMOeUM{?gDHN8mm*v-r5T(|P{H#taRbLrFq(>xa=@KC- zW!e$DdImhBn($@c>As^FP98AXu#-Ty1cpB4md#qUJV;RW<1QAy&C(_B5>*Lm8RD0} z$X`tpCH&H3anrOyfIjmMWm+s2)_R(!fi zlDN--D}9`lE(hh4HSridXDEaNvHWTYci#-23#GX~*zdj4$^W|Xf{U(rIKn~Q@p~DhWH#CQ zwF#E5JrLm&2vn5I?XFyD-Tc^4GW7f4hJ`@|E+tHks7WAl#|g>=il6S$jZN_I2WQK5 zw;zZpzOK+|uwDoq*5-*P^-LkGBJI1K@`w>n59nXQCaMYw5!aiI3h~^< zq8Q%eAP4GPI`HN*_cxCY$BVng_ZoZKy;PUD~oma~qOTEED zT}H$*ShlKQ^STvi{Zs%E10HXRP62FOL!hX24^R|Z-F5R3H&*rYPjDezh`|JA1)KMj z7i6x^DW33_qw3=b!k53rbZi$CtPxOCX#-R`^Y58GcS2wwsv-2J=Q zD1wG-%#FwRT?)PoiEVCX0RmML<_kTaax$7HDyP;f+>h4)o9A`vrV0|)<4nb>eGJg^ z*PE1VF;RaSYTv}e%rsm=PeecLi_asP^U1V|w{g#lUe?9&T7BwOz_H{@6f02aoCy_8 zJUcUcu-hedR-zwl=joSH;tl^?sX)1AXvzas2rw)ysrU+a+P;3U#XAy>8OS1#AGcjM z50^gHo^$k8^b~$0rJz=eSt}e^pnkz=@p?F2HeJGIV5uv4S`vx@kFqA-BcfiTNO3R$14?gbLN+^I-)9a+~V1LJNRBIW=1sh zBt{8!#flcxyX5vb5#0#^P85g{=T~d>sOQKEMim;C^C6QBS;s@dmz}9frs!fe+|MqH zYcPlAs?>Glf3T@Pu+^-=kHnfEWYC3jetJ)a63$d%HqFX*Qp@|YkNm`9Uoz;hQ{hq6 zQ{X77%(YZ*Gh#NgY3qT+tE1YKfad0p{yIJNEC>KdFL@VPoa^&z9%M#!2>s%L59 z0cJ^@9QRA6T6j;4TP_b+$5%A*m7qaEBJrMeLn}C*L ziQ%d>6SYrjK*+yk!l{#%>|c@s#IU!a$)b$gliO%oDl?ijwq2qjcXbhJ=`vjOMV^o)3?LLw#47o#e8aU~~6mh>^bBh<~c;X8n zq?j2<6|_28&!ML?!zCAFTbM?^iuov&r+}HMQH}(%R5gFV&>D(&K&3zJk@hMLd(WEt*suuMH2e4lz}_M_Fnob$A6PQ^L?A#o!UybGCJtk zrsj{AbIj^3@LMgv)YzGwnD0CF6_`ycnOdpgp>UbhuszUCJnh09bt;eu2irH4E~8{xP)5B26Mg z&0!4XLi6{ZtV>uvbWv^`cQ1nX;5w@d=N2aHj-a#WWC~Zzbkc+hP$Q&PUMYxu&O%rbmh$ zIIrRI11(U<`X#Z9&20n*&m+z;U?34O#6-Vfj@TU>2Z!L^f*7iU3WfpDBvJ^LhYf+_ zia|PA{-a`H+vvmOa4R{3ifvFJ8n>YXS zTj)D@Toz-@J)Q{wwq9fgc!hlhwcm;7cUTG0h+$^m`jN)|$AVGoU|>U*^D5E*gz*Q+ z+@xTb7BVbB*8f;Au_G*qxL4JJ;#XS!9UKIE0e2W5VH!VU(*Hp2KCl7F6|mm)kl2fi3gbKFNBbXZZzKa? z=9ZDx!T1j(CgTvppx0q3{>R$gg#p5{Pfc!n_#a3Vpa;W{JJRO=$J$|o5l~@vLU;4fi|3!Nu^RogF$rmZ7*liFh=4 z(LowPM^8_5ZILcbT0#O-(F$S%x}LRH%aW5*6VS6JlBC{E2!kQ`$Y?Rx z{t-9s)M>Pj)1e%PNHkXBLIrrFD}e)rcL2uuwd>KBblH-rQt?>m zP&28}PMRK_1EnGyxK>ti9`koLA{K<}c=q00f{BSqhFniq}H660QrD=Le-zpv*vWneD~h^vfj|Ih!y zvfwAz;!1iJ_xH{CZ2{ZifVh&B$B7pF4{>Dxu)aV_3s9E$ud;E58h|+=`!SRMZ(#oZ zgI<93eT96D{zZpe1J)>eEw1F%b+CT}^QU~hzDfpk2YMj-@b6Fm4LbLXYjKrb-S+4| zkSIU~h$~!*{HOoEc34S7KwQb$yaf)J`W=X0{`3(7#MKi~wKxB<_5^f5-({_mmiz|h z?>|5d1oWLjc;&7CSo{AYp#MQY+f|L>)UpB%Bd7?kVN7@s%J6fU`qJOXPlvPqo9eGe z|8lPc`qyo*24I>E_W_k+aNusVOG;}o07WQgtuWNk-nt!D4XbT zwgbc&dZ&Btm%aa+C^C&;OZuO4L0bdya4w6N43D?JCTqg+uhXnM$u~|;O)h*XmmqG* zYqp?41|&K$5`=sFrDCEiY`88EpF@WKz0|ST(LrZD8bXm#Vd5%C z-Uc1zJ_GonWWu_^*5)oMf`z}i+Ayeer8*xrVM`a_VvT&B-JstI(;dtL>V@O=x{Ifm zmV9=)aPRdp{2|!KwylUHL=@C(jUG(ErEq$}}w=oZzLm z^%wy0*U+SaZ?0Cv{vv2q$a&;1BGZypPA?l=y1V&>BvQj5mKAc_rK;*f0Bv-iEJ)!E z`h5o|Hj)6WA*uw*YTRH(P2?8N*%Zz@oL)H?vwPL(KEjkmTwA4f6$q{Em`UQ4va$N` zf&h<7y^68CAux|sh^L?^Y}f=JOwoeYM2XSjTQ-ZoOihyFQ_8rr#*3);-Kz$;GN}Tl zBlvfu%oxIMu;&+W^WT ztuenT)u7=x_u0*pmR0aKyY4tw;yBxCD#p$kaC((l1>c3*BN(XmmGhcTunSwZ8?2o+ zu8wi7ePz$~d_x5%QzyPQjl&9eP7Ej%lroVAH)s+>nG`Q;Gx)%`H9uSXooC;M%=)k# z_zm(VShE;Jw8^bFGzDbx9$Vk9DPzxn-gw8gl8=bPGL@Xke8V+?P;n^zU<=uVGxYva z?L*)NMl3?EH#8QAK8c`tbEXw&8wi`S&?`VF!4T(4--xxnQ$iwz2WyAd#~Qf~Q0`?} z-C9qaOMfCn==F%;NWpVr88AI25^z>z#AMj=UPPJUYhvF>$_oL&mW7lutlV^NQ1xMf z^MD>|4yVxV2QG@F$UHGSeSimILG!_s`T~BF?}V{Z62nb4?JU>QhaHj&-v9;Dg+gVoA@Bsymywv+OFcPUF4Q8LP>4io-*bvM+m; z@l7^v?QuHA4jroDn5Zn+z(w5pOa{;UMDlR@Yf~<@=mrkJo~~Jt4(RWch$;eWwGpUe zlPuW(Aa{*ZzaO-4<12Pbp&X0P5EV)I;cL_5UH1Qmyn!BQMZ`fQ*63$tW%j^Xqwq(iaAWD(>Px; z-SP1n7ft4c`k~JTsfieSCE2bLpN)=#z})Z?8v~O;8?Bp0UCv+9#S!aos{?}Liz>{5 z@b6Z_cA2|v(28PeH^YWNKfTj!dMJt}+igvp?nyM`E%Nd$+Sd*ysfoxK?UzU~PvU+loOc5il}zKnNnrZj2NK9kS+~FpU>EH5A)+ zZEeeCi|>XS!w$hf%GrCK{k{Y+5D6$HqXU?c%VBcV{5D@{f56ACNl|E`zz-`$|IquV0VwvGA$9qfa>K%l)5@nFI&CR^*Jfr;czNr!-TzXF{#rDwB(&`U~ zZDIW#ofLbuE&?iLyCOmHqXVoariZ=6iyRH5h9%t2!bz;xRGQLScauu()f(4Dn7MjO z=Zt|dWQ6n)bZ78*X%n_>rsnXU!EKGuzw+1?E?6h!sTxE&UaNkD(mJ4ySPJ9*>0;C1;12NapS?33nX09OjpL z|Bt84U{Y=LO6b%C^%bQED{k3hXLJYcF81V<=lZz$bgC^mCCbJ~_=>K5 z8bN=f8`=>N$xzt8ZYfpWXL%J~mPo@DlkxR~x9aF9KuCEmwx~XWjZ@0G1jSh+Q@8Ko z8(2Ssy`%Q&+PUdsoY?;PM+ftJ>QB;Aw_k2!=YzsM$?hui_Kro0czm4^oH6CeW z?3}>VtpBiqO^z6(0la7SqY7Eijje=Y-)A*`S2_KppcCvfYB*G&$tSE|XuIpl{~kPy zU%Gz_w{AmSVvJaC0+mwi1nJ-<^clZck!eNF5LVdCtqBqGta%Au>ktyOm#0tk7>M`w;=ma zLbwR2=p*S2V1w3;rFVfv9vKeqtBhHTFI4u30dk>MPIVsZrUMo0{s7J*_*So2DTa7P zifBtIjl4~aA!S*8k>vIV1?lxqg5YsET@#+S{f&bCQx8tiqG;}^k(RDB%5%Wy{{>K} zje9rQ%`Ntca<{Ejz((kGnCPnTokK(LO(#?0+MifsZOO?^edAtkLvS}V0F1B^e;9cO zFa(*x9fU#vLjB z62s~ab_O7584eq`Nm3$+(6L+m)~O;5t``Csy_RNxA7SXT|G*d_^)=XpGZ4`^mH%$& z?Ftx^F}h|(5DSI~^JNoy_GsRO=dV%$fjGnublTv+tlxb{Kku3Mz0la$*nQ(joCu%Z zX9@dD57(fySITrb!})J>b04h$wTH!osyRyr;Aa`lex)np8*f1bn?@|#Un7!QR|j)+ zPj(kVqu-KDa@tzrc+TC2)9XsasjrsJkO&P6!}t(yV`nJ8F`1ClF;<^I1aPSSP)RhS zwos)KT`X%9ws`EAf14{Dv>}Qv;&n2=;b3+|DG=gm#>%pz8Dmvf?yeh3$9V(&7^)^Td+K5S z_bzx>Y5EB${m0?qQ|;|T3kgD1jf@RqtnD|1ljM!T)Pv4>MFY z1e9n^dqE}gB>n! zQL7h=?$=!MO7^ghS$BSlb@k4$9cqtny^dPtHn%R+`3@#*AhdS?Sng`>~^{>kMy7P zyj|yU6sgM>*VQiQMl1MA4$2~xsy6pDK0c?Uj@$Rf>`}sIR9(5dhG*hv1evbSYr984 zs__8=r*$1|^T!_kLx+Ro5ptc3LUSEnz23RyY!T5j%;t;SkU*#9tgEjpvYZ(&(7htJ zz~k%#ra=t9tdkky=6lK-5U*fT-l2fgc8cN}>G&xkIzCqP#KOr3zu2kl$O0BHlc_Yz z6o%>eW+()4Mdu z1j+b92bm*9N|5hGmEOX|e)%lP6dtMJR9sf-4vp%3S5f)m3#CPmM0;YV?!lXs zxrE1vq*C$VozkjMyVjaPb?m(*UD~0})ic9>#t-o4w?M?OW-ypUH>v3%ZjwTaCQIeB=2VYGn~T;4Ml1z>PYAaPw9L$z zT9I^ zklG0*!Jo?**_O7PQqa8mdX%MLy^FyBm6GLM zfIbnTj~5eOJ+8h^G2}k(s!7L79K{x{me|yqS36IR<~W3ilL1xa^C<3zW&HgGP#xs( z&q`F)Lc+!6vznKpLn-$6Kv-~ou5q$o)siK`&#Of`_7ILKI-;!E5HTOXhUl2+cJAo{ zQm5S!zh|cq2!^025-Q*Dn8A&bAPOsN$};wLR}C5D)(5vVnZYDJvF9)?i3g%t-vRrM)9pC$Xy_oM%W9=>o|it$bM9pp9V!x} zJq`l8A4wCfs_eS!78I(?n}9_G=%F18a;%#sEb4ZQ`wliQ z9>|6)B=Wp9(@et)h1t z^c7Z%0p`C4Lh91Piaa9{gRSE;bt4!Uwh)NGQ#r&eQS>rIozm_y0t#2Q`01~{!iP;5 zK!7U=of>pg8oHup4(B___`ezUg%kE zD)wVo6xAwAU69OO)6lz)rqKh5$Y*s(hV8{|D#50bXU{!BW9Us$) za{7xjFFDD6%E|1tJ9T8<~!|g!t z!UxR=%`VC!geS|xf=pA1gq?IO)#YEIA!`-aV#!9@kMOGqi)c#-;v-Q+0lw@=RPi$=Voc$p#N0!&^1HW@veKd1K z2KY#1k7qX#lZe94a+p= zA1H6Kha7d18QNNugy%b^P1bI;grwU( zPdKbJ+YKfk0iwIG0kD9Mb9kw5BtO9f2nU3>y&tIn&h6EJway8Laz>L%wAe2 z5TGAfzX?yO=jp+wg#Ey@(f5IT%WHTpJc&6v?xG^X%X@Q?yImR{@WJ3iRpQ%fw;7aO zcbLmWQ;K6tY^P*!6HDE6iL3~rNPwj(5B0#fi3DOGBhgjBEwpOZ|1f{F%^;GFE}ui8 zub7Qai)gq7atDfwq;oW`5ut+&YRuJAV6*7oO;cpG3#cdgyw zp8@)tVKrLjrV%&80S$;0*YSU=FPf6_s>0o>yPhZY;4gi#y3Gw9eX!RXvJj_<`o3Am zxsAhsF`Ho){1LJ8%CkGYiA1b$t1?Mn-6vBA<0(aj_Tp4@$(#!E(z>24p*7}5XqlTy zAU{ww_m^z{ouRG}m#-GP;y9Sq3hxncikO<3_8}{4$dNW*nLeDH>_K^SYU?%MzzXqm zQ^mAlT(dDH36vaBwY)dMvFcQb3a!SbQm+=>)r_XX`*evet590U#^IpZzB$d6epY!O zDu5hbpiaLQz#4ybc*MvbTh$eH42#=7($RmjL=s{E-8~0Ut6m0& z7to_pBSTbdT_9!&w9zKYC?vgysfqWYU%W`ERm)X60?O$=ZS9ucItXB$6k!^#5dHiY zPY4ZcNQ<0|o>nM@{7q?u3MOs>eNsVOp~Dq3pF)u+q-@W~1v}2+_tFE2DbkQ31~I0Z zG7Qd)3z?KG)q9tLA=Ng|x^L?8UtC3?bPx#aBD9+P!GJ%#ypU?(9+?W$Hel%JO?&&t zT?0}8z2E&>dH1`)@XJfSJJ2$Uz&)!&cR6lG#lPPG0D8^XL)*NjH*X=90{LUpM*@A+ z9h!SfV$zTzYvbXkV--fUc`V$nOgrYnXkp$TtKx1Lw;<(){1iG>9O z+$o{Ls`ut6p{Av^iaQc@Ac$W4{?2T`J(ZzD!@03CIOP`|cGDeoN0anhE({MzpIwgv z^%J8HW0vH5rcz-_uoEN4cXH^IvT*#KQhBs?zuAAjUiwI-HLJP=I6i8${wS!JiN2>p zs~jyLhR4OL)MWeF7I4ze@u+5dC)JGn9^mi=(_J$M3;cx$3lJ zrgegj`|H#tmvLD3B|4x|gDJHHms#poGGt`rh1InJi#TSKbcqY;O}5p6`bq1_1311* zc;(nc`mseWv$}}&2|G3py|(|ay)Tc4vitj&C6!9<7P3wWEtW)fgOnwqvTun&mXK{^ z-?DVuNs?tqwrtsVQz=AdFftfs?2IupvJNxDb9F!8yZfu(^Zfq$y?%dv{lScxbFOpF zb*^(h=lywq-X9N7{OR=Gew2P{5WB+N=JiE!rbLAs!hxG5i;Fy!Zrsu?{f)^Rb%EQf zFC=bzczDQ-JVN^RLS5}PRQ3!rw!Y%@s(PHT0Rsi;#(s%7@DONIQawtmh%{jbS3jO2 zg;dlOZ=JQ3w_O2^7T`#Oc`lEd)<`X%*@(H-%>$r%Gx!euolBGn)Tt*saY%z`U*d-2 z@7HVkl$`hC1OV&)@m*E;w@2Z=?&Np0b?TMMijn6-xoS)$K-_lZas}qS-f{5qU9)Cy zvq+>|vRqgg(?PC$cRgRF$uzWeIuibBQLe#bMnirfeC;MqMyBC`vTxFq$uTK*Cc2ek zi0^0+`blO(VzEbl;I-gZUQddRC*H`US!G%-jM2=@Y_*>$Q&SwDzY2sTTe!i?2aA>r z-8rqHO*3mz^2g@tE!;e^*47TFJyt=B+pB{V0P}Z1-sANCr%&|*b4A;fzvgpR-oTh| zv<=xylt<;K%B@7$9FS=|m2C)*K3wdXuRvC8lH5XMM9>yivwbJ<&4$ku3+K~w7-N3T z3BQI~7tFB2aQ2!+^Ih-DB3=P2Q_-{Ocz8zi8v|?Qy>B)5;lV)!%e%g{fgvoC})Z5(9U3cua45+b#yidh6Wxz7J)OIWQzKFo8 z{7%IuJn`SD60utu(2Pw`x_IeQvbJ@IkjYU2VffyU=vxH`Qh;Q=H)&1p#czMi2Gljo zj>O8ccI_;;#y(=S0VKj-@GEW}ML*YW(S!%D&OJlkBy@}eiE+dx{MS$LWjMICaLv&+ zdn+#}3v!XweaGIFikeq{7HxMtK@ApU7z!#SO?fp7cS|9`f(=-oBA;ctp1tw1 ziim=pu!OPK%8%;PbgdHKS)^ACdt4?j4ki0yy?pD;Xv7A50;t}vgCGGv-^Cvk=eLL_ z|Go9v_m^zX_hoV-+#B2`?r&~gJa7Ujed*eMukK9|z!+C3dRO7U*THVz?-5b1BfXNx z*S2|FW39toI7Bd-*Q3$zqFLuDGo5*fI62+n`fA-?~ue> zHt~W1>qJhEb7*QjOH1#bm3y8pLB@0Nut*G4p>=mYb9Hmk*FLQ3?nVruA~lk2uYR?^ z_+CX;z_8hHRg;0~-HVCfCckI5r63*zDLv)BL$CUh62ZG8t#|y1jrqLqal6A+K?`VH zaH>T6>kDw&HQnG{zt_@kgQoRZf0xmK#J1VSj}fIrhLY!D^lc-eb>O!B>f*`F-Xb<- zec9EgAB8x37f#CRKd&<&XWR9pr=B0aXN0G2)U1iO-p%nRoh^N;P>nC-kTvg`N2CY%mn{$>j6J@FoAAj{``J<*lRf5k# zi#ARR25ylfz4DnkJi)g^RsXj6l-7Piu&DR&on2xCcdLR{g^u2>A6o5nVMK z@Zgsyx3$l;Ltxo_hw-+f9hI%OtE+4H7V_mUN?b{ghCRI#WgQR?ITcRFp6lWlL2|iI zJX(-LRiZC;9SL2Z2=YCY^&A_0B#%9UzSInCFYZ?PiUJ|4 z&v_LVwFu+(oK~WHeDb%k{#$(V!He3%+d2}hS|pyiEUcEM{?)U=#iIjCx124bq(#o9F2d(E$t3bp?>hL{`_|1)g2NwEA95dLHaMp#ZLAwy`^Ys{ zF%nvvrJJ`aO)>iv!scN{$ag5P2*sR$NeJX~Sr@<4bY z+bdyljEkhp6eSQxzbn1CHG@?7n({@bAYhABt(#ar_aaZ-ohSZ(;IuN#i5>)ESiY6ly3<=V z52Xi|UmuH=h&+SH;56F4Eqf{|0qmJ1$miQUUo{}lYu@yPdBSyeZ_+rGoho&@v z$sa^+{o$}Uds5#!D`YxZlMnA0#Fd?!;36Ef=||y)9O3Bl12CRO-tBJa5!-$AfI)#LB^PLrYSxfgPdPF4pyz(Iri>;$xke!#MU3SRl z1WgyTP!Cu;o6qMV!Mks7>~tPt=~C|0dhrAd2$zuLLWu9D5@yzr@(nl4%}oHJS&dJ& zr2m!Lr&!vGkA)A>IU5})w(X)8Ne$U0MJI6>dRRV^h$!?O{LvM?x@K(DXach`h;YP= zbO{8So^wZAUKf3<;?fzz+x4jFn3sC*-UD7q*Z>&mf~G5Mh(w1CXLEc$?rOTBAJ4n) zJ4xT#m9@LWZgS45#)?aAxqqkiPdp|hnt=TB=PjMfv_Rm0NmcyGg06xwR_7`O;`wJ9 zI}h1LSC)``8;4<4P@~0pg8LyS%=NY9T{q?`b%9o?=1*Q5!H`pSi)AQ8@%jVZEo=+< z`BQgYOk%U<4z&cHzj{U@mQdsTy6Y*{ozTGsdimTROty2h^qpV`YWs` zeOm`NmUz5!1zfVx*<08UoJX19Rcs(tj9^JHE`Lj|XtC%rBDPX`dROP(Y7Yx(M8D%Y za3*7SM%s@6#ESKEXqv(B7GzC!dA)VN`7`=VQhhz`gqW}^5^bW>T|rR1V7kew*=HU` z&uWE) z`u!$kaR(sG3@r5YhddDaNx>N6#lnOKE(7i5S$?F^DO_Z%DGRAo{ds){ot0m-)HOeK9PMbnkQEXf2}?ZKX|AXqcZr#Z>6%fg5{TMT;i4wbj1NQ{ zlux*#*pKUC6$maK59}hQ?UoM)_?#NfJ=fJ^()n6tQ@6Kvts-a^zW}R)L6==dE1Kt* zR$fFj3nUWfG438T`jE6w?J#RTF4dv$b>|GnTMh=mtUOR`_vc>>n_uTR(217NAI7ap zkMEpCeK1AkR}+yyVvYlX4?R|FqI<+!h~{}Sk|7fcO)G;sWtawxL*@!2KAJllt`LOC zD9YvsySbHpv2oklB_9}|o_BT7B*;I?5$fcbsD6l0lEiwyJP^%$o=Qer$H@}PF`Xd_ zYQysj7F{Ydx$gs#zN^~m`Q}H>P>oAVHpU9gAX?E3_+^crihBJ7lt+H2JAd{&r}Yax zK1Fl15UnOkUtUAhg$tRp`eHpx1NXk$ZTnAidJb%J znr06+SM-f?W-U0Tl5PqFtsXYpB-conyT3B_osV|sc z1?v&mD*44yI`g1B5>B~MRq6Exg^J&EEaS?z{WK@3@zzw^DSF*YgDTtbl;;&*V+Xuc z(Zc6KD#p9|PRunLqB#{@7cV7vsmXz2h-GtQ2dgy415+W+YLE+d6!-L9dK0}cvj7r( ze3%J#wXhV0UvO?|+u3pP>{8y6OBuj1k>S*ex}>%l_2VasZ{|+J4_ z4|?9tS1Blb854(Bovn)IuUOG&DD?W2)E5hKVI={!c0JL*FYu|%>b$&xpGfe1@ zhJsI46J6V$b+Gaq+nlq9ES>Qi=i+$d68*}WC6Z<8#UD14J+_g+r{k?pTIuRR>fm*9 z`0D1G8V)V{uB(-v(5M+cVikwijoY{iPU)>^y`kXRmd0_4NBmY~BJIl7hzf6%mtre@`CYa4BOts2mndJCKp zr&i-ydq*!RLdWM}Ay=?DT2SY(=O=Zl-G-AwEnrx(jak4eBpV%kbJoq0a* z?nVpD{wR`3j2U8}N$D#Ib=JmAam4&QkF@R~=I!~a(A8Ejo z!+~>K(I-DW(A_iucY5bdIl3QT0j)49xFRNe$1las?+O{fGtTb}ZvZDT&`wlU0O~lV z;~<|Rf={}eZFw}9=L!i?(j^DUP%6J(y(@_By>JPR#IMk1lAR_w8z3Xk2cd>U4Z_v?%f*cV+Xk=arF$iF%Cr z)`d?WZYzC4-0Eap@AN;!0p@6$9AC3r$o5U3nBCR;~Of0@ymsN#*o)y%2R? z{CkBi#?h^&Li<@*TBD1m=7p@_%?u-iN67a3D{~+S{h7FxLJMjBuI5o*Kg!gU1Ygto zm$wqPA~Xa7*KMBa9&6}t)Ca%JbZ@!N3mQtkZV)ooIZBv4HoUwicqbg_+eP1Uq!g8|BZ@k^MX0BYyV$|vfx6Or z+jD5^@?L5E9Vs$a+InjopXGeJ5@;uNlmy9sqhGXya-u9-q;|&7&2z|(>9as_M{RLF z7-Hv8AiVo#zLh>@*#Pov{HCVjJ&;t4a>(%V%4^AF#ztftC6J{z5OYzU{$56H`4J%6 zblW;&M4>O&?du_17Z9e6zKZegDlv%1Rb^zHwr1%65E*3g{$bwwNOLsZu#URZGN}c5 zr%WwYZ^I`su_sWZOkBe)LqpH0GCXtmXD|HkSE0J9oUv^y)jGVfm#yvZP*H6dW5L|0>XiqOY1nS@ zxyW<9nfy$>qU+n3OSa^p?`xa4dN&&1=_&>gEo`d1#ssO4k~dG8uoJfqS)jPVF`R5Z;*ko@r2RfgziXkIHV=L(S zggaHi1{+b996t$2cASXc{3FbUszoi^*r%m|WA2$aW}FXt&+t&_E2ZX@M)8y~g3c?< z&PL`LN!;*tb9GK-bmiFN@A2dPA7alk3@ijnL(2kXH&_YD&8AHiUr!K+y{I_}It$-& zh;lOQyA()N>oy{DK5w}VjjhJD(}g$=4y!*_G$`7==!;YAfogn0owD-AZdZSJb$=NQ zu{z;}V)%X%0Am)ijXV#D)>5x zggp5Tn`n;w+WglW5N1PEp>DZXbG`~EtH6__8B2Rs`@HHH0lcBBk*g*@w9(RY+kRLm zAIBr6FgKt`cFpN@p8OcC#fEt^H~fVIy3`aN#B)UEX5+h61Kl_$__Br4(Pg|*9V{8Y zJl*S@Wb}QmeK!tNU*qq>^Vx0SO##iGcKmun$HAj7{5P_~Jl3SBTb)hu#OUuN+>^EV zYx5mjeCRouZ)uX`gfm;nmow)93qj+^9f_jfEi$bZ8w&+rhywaI21$T|vQC zBrZXe;@@M(ssjr_qet8_`&^h7SpaP1)@O$9Khv(v0Z3}iV1ELH7*p}J9ss}w2NzfR zEZP(TsZvOlN4n1oMD%3NB-ZPz8BR!z+drL}OaH{C)`Du>s8HmMZ3tNZ{?WuX=Ai_Rok4q&}1rK{&o)v z>HdK87Mk>gRdxB+cNkLu!%dYd(u^$4AYWDp9|s>xo&^&`dJt0y=f_s!L#_>$lIoZG z+k>7?gh`0#9#NSld`+Fp9v~}dcpR0lpH{e#HmAuXk@fGp4 z;z_T1qVna^RpTgRk@joL5Tei2J<%UB#VZf>46mr3z-Sv{Ku|It^k}7^ZK1*{La{(I z@ijf@8tO?%$d9@7E23Ey`3>N!!8t8H_@(wN2(#@UF>arbuTo}hfA{vZiap7#(s&j` zh0Na>CFe5DDMOMUS@EdJW?$JAf%HY3)RQ+8*3YnoTW&64Bm&1vO(aGuLqZ5OCV*fl z_Nu}(F;nm4HOjMdo^#(jI;>>@GxDb!B3SzlwONjONnO95G*EZ@8y>c9CyKJhZzz(7 zEToHV-I^1gR|EimhQkxiKc!Ce#oi_GbOi_xOdVp@_CH&10ct_0}u`$`XGaqYyHV+UiXO#B5k@?ye?tmkkIWn@@3^S zz^^gJC)Wdt8uCZi-Swq0vYoHZ9%YBPMe4u;*Cj_?kbqj?0yAS3KrVT16i4{Y#S$oy z%cf@SB_vQf%0&c%Ki`WA5NNbkfD+D&z=Ge^k_WYqt2iVHc6;KA1B9*wpe)v&%k-C2lDOxx zc9!B$dZ~0OdEw&VhXYKrBanlNh#+8cu2Q7*6%i~%Rn2>k;S*~jb@D^hcmV%ePJ#2$!9yIhyG|AB>Xh=U!>kDI&kKGim4`h;C#ekFj@DYkw zO?;1JW$YYvDwa^_Zf#0aDCg|`k$K@lS1|ZkUAFr&M=VC{@?owa-c;I|s z+rb(rCV-vr;8;bVv_qVa#_gs?}UQIfFIk4RKJjLi#`I|J@)%Whb{U>&mNGxCvk2Ogq8 zg5crE_Dutht@KJ$AXmNn*&j8I`lJ>lcy+W<|4o}xL8VS9?2=PGr|^A+8jbnC`0H@E zP76R^Cv7;~7uw5IpDjrdT>UL@FjV!(A;+Qx9b_8_r@JGp*8^!epdhpQPro6f=#YR%qu^W;w|f!>>GB(#4-HBH$-FO(zjb6^HNf6pa;ryjCe- zH=AFWX%;n;$n!_lQRRk-2VibxVcQjTm=jh&KcYngOFR_3=*2IkQEFJfSO=v{kdG{1 zJ^k|8!pNTEH|p>Unx0)-=mO2a}@UM>PNGJw_gEMw5 z5Xm16+}kr?$L? z1a~EqzKY~rl(qDVzdm-msA~{*I9pslLUmJj@8zwNwD-6UFyr%651F;j`)Kp*vv!J^ z4iMvPn6+qAKD!h+r&vm4=cn#5o3{jS?=epu{74>zBDUoSE6$gimoVkZF)jk+l=7LH zB9iqq2XZ4#WU_k}DL9#M{-Y0AKSxnaCV#GmBiIzp%v(Pysqk8IV zu2;x}4MjoK%bX~v4|$KoUTI**ZahNn2W^VUj$Z_@ROQQ9Phx6u9r4?=F z8i{_DH_l4O1SM8|*gnTNITPF&Q|TzH5h_=} zbzpiu2F-CG>8WIzf?&n~S<&<~{=X5_pvFB1_%U*KmP2elo zIAihoR9UZegWhh|M$v3}Pb#$2(2QHOrcz{*<-FB5vxpv-HS}#x!95G;__gVa#XUXK ztPwpu^95_nRkCPfSKzD|?tsW@s(v`dD0pb(wgPl{@|A|$TIxvbfAg*ftabheT^%Lh zeE9j~_nX4M_In*b6yz2MnvWb;8Shf1{Tn*-H^<~J)&WfvD2%Mr8U2le@NaJcEa5;> zsIk%)jeRBm?Hj5ekihWJEL&du??juhP{1lV?Dz!;JpA{P`>Om;WBvzz(qE(X|7K)z zg&Fhi77I5B(fJuc+9C^K`|u%ETNWUW@PeSAAT0@a@fjR9^lL^jGm1RilP%mPBMfi; zPqBz$=0^bWp(ulqMh}RX!Mjl+_cNP+TtG4RKS1`N*O)+fJbpM<_TEeBl5`FR?aQ+4 z|9t0X&r<{+9#rk==}B2r-7>gF!Rg8#?)tllK%9q@2I!+s=2-^Gy}pchep@e{-(60T z*f*O%UznHBl>&%+Ha0dBv!dA?ARi{A!M}R(!G+N&BePI^<9>VWpMQQD6=Ik!Fc;jf zdIUt;a?3pC64&agJVuXaq{I*>@u)aFzLsDeSJ3bL}Y1z*k{ zNE!UDKEH2ThrK>5?L4IQ%}4Iro?5xt?YhR^2zROBH?`xx2o9e`bMsYNW$0KwW`_bq65b;5*oUq3VG_fZAZ{0(}CoS`kJ znM>Sh?7>lz@asAeLq}6u{?&{R0*swHm$@lg$ZNwcYm<(n;o*#uOdue=qVBfBj|WdL z&1ruH?W;YBn^8ferh=u~rsJSlopajI+}_)$2!Sa|^bdGmro$brV;4Ms*&r8mKCnuJ zmsvJlEBg+xot%L7RL?roc!A*R!WoUfyZ6^Z0oq_OuYY{@c<754FS`FceZ?+TtiG6f zhdLRuOW;rD{l%91dYZBT3%RKox;XsiR_rXeHA)|Hq>THM$9_wDk|iuFi&wJf)8IXj z&-K$UIm@~QfAvGv?tm2~5D*caV)uDgwN38rHYq}wF@j1AIA@vnZPZ#Zsx46BO{R0r zVph#V{$3L2*ah$X({Ag-i&{YD9?s}q!^#KB10fdv_JDsn?e%%s-*cPc=S5Y;0IT0n zN=mjby!GKnMza>EtJwV+<6rdv(8r>==h^}?eRXy9i3f47aQ((M>PF{$WL(N`akOD$ zhZnQ@V$X=BR99C&xzP;PDm5;5PQU2;51WZ{UlmQeTGTek=)BT##lSk_^bLm3FDCcL zqn=eEZ{G}TC^EMxf2^IIQ z*aO$Wo#SZ#T=(bQ_V~lJ!7aPYysn4)n5}TB+6g+$%1Y`c zj?&xR4Z3JMz49ykW~DK-{iuPtxpNS{`{A+RSA;;vA3xU$exdPNCj_wUc{YBD-GIZc)5U8hWK#WTd(B6@51CH}uJ@jKqkz^L{!^z^fN(YJjie_iEg0g5GHc_6R6n*P5u<|g3b zRV(x9iQ8B5*P{zm{-4JDZ#UsTBlDk;**5|w{_`~L8;^fht^c1RGr6~<^q4K8vU-96 P_|d+lceCQg!|?wC_y&s& literal 0 HcmV?d00001 diff --git a/website/redirects.js b/website/redirects.js index 8e31b56f3401..74caeb5e3885 100644 --- a/website/redirects.js +++ b/website/redirects.js @@ -4,4 +4,16 @@ // modify or delete existing redirects without first verifying internally. // Next.js redirect documentation: https://nextjs.org/docs/api-reference/next.config.js/redirects -module.exports = [] +module.exports = [ + { + source: '/docs/connect/cluster-peering/create-manage-peering', + destination: + '/docs/connect/cluster-peering/usage/establish-cluster-peering', + permanent: true, + }, + { + source: '/docs/connect/cluster-peering/k8s', + destination: '/docs/k8s/connect/cluster-peering/k8s-tech-specs', + permanent: true, + }, +] From 3117db29354b3b2ccbaed503eed518b1ae89bfb3 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 10:53:39 -0800 Subject: [PATCH 039/421] Backport of Troubleshoot service to service comms into release/1.15.x (#16395) * backport of commit e8466bf47933e1292a770a66add5683243d7b264 * backport of commit f919a6e77b8f5f6ba964e671873a82c2dad9a903 * backport of commit e5527649ae2c2c6b663dbcc813c69744609fb680 * backport of commit 5c40ba5360dfcb27b501da44a7c075968f0a4eb2 * backport of commit 51b6f5009f7408e13162f8a9444fd98b8bbac21d * backport of commit 00ec4e5ff3b1b18ab23ef0beedb4cbdfcada22b8 * backport of commit 1405edeff9d303ac22bbc8f59c8812da9920297f * backport of commit 42e93d59f68926c64b4f23f77e6c4585ca6d799e * backport of commit f867d2edabc26f71f338d6fbb95e85f1cc341a86 * backport of commit 948227199879c9451f248e8acfd504a4899092a6 --------- Co-authored-by: boruszak Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --- website/content/api-docs/operator/usage.mdx | 4 + .../content/commands/troubleshoot/index.mdx | 2 +- .../content/commands/troubleshoot/proxy.mdx | 2 +- .../commands/troubleshoot/upstreams.mdx | 2 +- .../troubleshoot/troubleshoot-services.mdx | 150 ++++++++++++++++++ website/data/docs-nav-data.json | 4 + 6 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 website/content/docs/troubleshoot/troubleshoot-services.mdx diff --git a/website/content/api-docs/operator/usage.mdx b/website/content/api-docs/operator/usage.mdx index 73d9adbb7bf6..ae1e9d87ccb8 100644 --- a/website/content/api-docs/operator/usage.mdx +++ b/website/content/api-docs/operator/usage.mdx @@ -50,6 +50,7 @@ $ curl \ + ```json { "Usage": { @@ -75,9 +76,11 @@ $ curl \ "ResultsFilteredByACLs": false } ``` + + ```json { "Usage": { @@ -129,6 +132,7 @@ $ curl \ "ResultsFilteredByACLs": false } ``` + diff --git a/website/content/commands/troubleshoot/index.mdx b/website/content/commands/troubleshoot/index.mdx index 521981a77e0b..0c992aab15c9 100644 --- a/website/content/commands/troubleshoot/index.mdx +++ b/website/content/commands/troubleshoot/index.mdx @@ -9,7 +9,7 @@ description: >- Command: `consul troubleshoot` -Use the `troubleshoot` command to diagnose Consul service mesh configuration or network issues. +Use the `troubleshoot` command to diagnose Consul service mesh configuration or network issues. For additional information about using the `troubleshoot` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). ## Usage diff --git a/website/content/commands/troubleshoot/proxy.mdx b/website/content/commands/troubleshoot/proxy.mdx index 6c93581b155e..d9749c0c254f 100644 --- a/website/content/commands/troubleshoot/proxy.mdx +++ b/website/content/commands/troubleshoot/proxy.mdx @@ -9,7 +9,7 @@ description: >- Command: `consul troubleshoot proxy` -The `troubleshoot proxy` command diagnoses Consul service mesh configuration and network issues to an upstream. +The `troubleshoot proxy` command diagnoses Consul service mesh configuration and network issues to an upstream. For additional information about using the `troubleshoot proxy` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). ## Usage diff --git a/website/content/commands/troubleshoot/upstreams.mdx b/website/content/commands/troubleshoot/upstreams.mdx index 752bb0463c51..425ec39e4642 100644 --- a/website/content/commands/troubleshoot/upstreams.mdx +++ b/website/content/commands/troubleshoot/upstreams.mdx @@ -9,7 +9,7 @@ description: >- Command: `consul troubleshoot upstreams` -The `troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. +The `troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. For additional information about using the `troubleshoot upstreams` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). ## Usage diff --git a/website/content/docs/troubleshoot/troubleshoot-services.mdx b/website/content/docs/troubleshoot/troubleshoot-services.mdx new file mode 100644 index 000000000000..3451d2e50672 --- /dev/null +++ b/website/content/docs/troubleshoot/troubleshoot-services.mdx @@ -0,0 +1,150 @@ +--- +layout: docs +page_title: Service-to-service troubleshooting overview +description: >- + Consul includes a built-in tool for troubleshooting communication between services in a service mesh. Learn how to use the `consul troubleshoot` command to validate communication between upstream and downstream Envoy proxies on VM and Kubernetes deployments. +--- + +# Service-to-service troubleshooting overview + +This topic provides an overview of Consul’s built-in service-to-service troubleshooting capabilities. When communication between an upstream service and a downstream service in a service mesh fails, you can run the `consul troubleshoot` command to initiate a series of automated validation tests. + +For more information, refer to the [`consul troubleshoot` CLI documentation](/consul/commands/troubleshoot) or the [`consul-k8s troubleshoot` CLI reference](/consul/docs/k8s/k8s-cli#troubleshoot). + +## Introduction + +When communication between upstream and downstream services in a service mesh fails, you can diagnose the cause manually with one or more of Consul’s built-in features, including [health check queries](/consul/docs/discovery/checks), [the UI topology view](/consul/docs/connect/observability/ui-visualization), and [agent telemetry metrics](/consul/docs/agent/telemetry#metrics-reference). + +The `consul troubleshoot` command performs several checks in sequence that enable you to discover issues that impede service-to-service communication. The process systematically queries the [Envoy administration interface API](https://www.envoyproxy.io/docs/envoy/latest/operations/admin) and the Consul API to determine the cause of the communication failure. + +The troubleshooting command validates service-to-service communication by checking for the following common issues: + +- Upstream service does not exist +- One or both hosts are unhealthy +- A filter affects the upstream service +- The CA has expired mTLS certificates +- The services have expired mTLS certificates + +Consul outputs the results of these validation checks to the terminal along with suggested actions to resolve the service communication failure. When it detects rejected configurations or connection failures, Consul also outputs Envoy metrics for services. + +### Envoy proxies in a service mesh + +Consul validates communication in a service mesh by checking the Envoy proxies that are deployed as sidecars for the upstream and downstream services. As a result, troubleshooting requires that [Consul’s service mesh features are enabled](/consul/docs/connect/configuration). + +For more information about using Envoy proxies with Consul, refer to [Envoy proxy configuration for service mesh](/consul/docs/connect/proxies/envoy). + +## Requirements + +- Consul v1.15 or later. +- For Kubernetes, the `consul-k8s` CLI must be installed. + +### Technical constraints + +When troubleshooting service-to-service communication issues, be aware of the following constraints: + +- The troubleshooting tool does not check service intentions. For more information about intentions, including precedence and match order, refer to [service mesh intentions](/consul/docs/connect/intentions). +- The troubleshooting tool validates one direct connection between a downstream service and an upstream service. You must run the `consul troubleshoot` command with the Envoy ID for an individual upstream service. It does support validating multiple connections simultaneously. +- The troubleshooting tool only validates Envoy configurations for sidecar proxies. As a result, the troubleshooting tool does not validate Envoy configurations on upstream proxies such as mesh gateways and terminating gateways. + +## Usage + +Using the service-to-service troubleshooting tool is a two-step process: + +1. Find the identifier for the upstream service. +1. Use the upstream’s identifier to validate communication. + +In deployments without transparent proxies, the identifier is the _Envoy ID for the upstream service’s sidecar proxy_. If you use transparent proxies, the identifier is the _upstream service’s IP address_. For more information about using transparent proxies, refer to [Enable transparent proxy mode](/consul/docs/connect/transparent-proxy). + +### Troubleshoot on VMs + +To troubleshoot service-to-service communication issues in deployments that use VMs or bare-metal servers: + +1. Run the `consul troubleshoot upstreams` command to retrieve the upstream information for the service that is experiencing communication failures. Depending on your network’s configuration, the upstream information is either an Envoy ID or an IP address. + + ```shell-session + $ consul troubleshoot upstreams + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +1. Run the `consul troubleshoot proxy` command and specify the Envoy ID or IP address with the `-upstream-ip` flag to identify the proxy you want to perform the troubleshooting process on. The following example uses the upstream IP to validate communication with the upstream service `backend`: + + ```shell-session + $ consul troubleshoot proxy -upstream-ip 10.4.6.160 + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "backend" found + ✓ Route for upstream "backend" found + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + -> Check that your upstream service is healthy and running + -> Check that your upstream service is registered with Consul + -> Check that the upstream proxy is healthy and running + -> If you are explicitly configuring upstreams, ensure the name of the upstream is correct + ``` + +In the example output, troubleshooting upstream communication reveals that the `backend` service has two service instances running in datacenter `dc1`. One of the services is healthy, but Consul cannot detect healthy endpoints for the second service instance. This information appears in the following lines of the example: + +```text hideClipboard + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found +``` + +The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/discovery/dns#standard-lookup). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. + +For more information, refer to the [`consul troubleshoot` CLI documentation](/consul/commands/troubleshoot). + +### Troubleshoot on Kubernetes + +To troubleshoot service-to-service communication issues in deployments that use Kubernetes, retrieve the upstream information for the pod that is experiencing communication failures and use the upstream information to identify the proxy you want to perform the troubleshooting process on. + +1. Run the `consul-k8s troubleshoot upstreams` command and specify the pod ID with the `-pod` flag to retrieve upstream information. Depending on your network’s configuration, the upstream information is either an Envoy ID or an IP address. The following example displays all transparent proxy upstreams in Consul service mesh from the given pod. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod frontend-767ccfc8f9-6f6gx + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +1. Run the `consul-k8s troubleshoot proxy` command and specify the pod ID and upstream IP address to identify the proxy you want to troubleshoot. The following example uses the upstream IP to validate communication with the upstream service `backend`: + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-ip 10.4.6.160 + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ listener for upstream "backend" found + ✓ route for upstream "backend" found + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ``` + +In the example output, troubleshooting upstream communication reveals that the `backend` service has two clusters in datacenter `dc1`. One of the clusters returns healthy endpoints, but Consul cannot detect healthy endpoints for the second cluster. This information appears in the following lines of the example: + + ```text hideClipboard + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found +``` + +The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/k8s/dns). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. + +For more information, refer to the [`consul-k8s troubleshoot` CLI reference](/consul/docs/k8s/k8s-cli#troubleshoot). \ No newline at end of file diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index d5daac6b29cb..219ec663b626 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -807,6 +807,10 @@ { "title": "Troubleshoot", "routes": [ + { + "title": "Service-to-Service Troubleshooting", + "path": "troubleshoot/troubleshoot-services" + }, { "title": "Common Error Messages", "path": "troubleshoot/common-errors" From da95252505cd3f1b58ec6e27a50ae3a6ec80aac9 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 11:39:12 -0800 Subject: [PATCH 040/421] backport of commit ed697687c4573b4ed9f65fa30a4f1c76eb6654c4 (#16402) Co-authored-by: skpratt --- api/operator_license.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/operator_license.go b/api/operator_license.go index 14c548b1a354..74eed3baa4d6 100644 --- a/api/operator_license.go +++ b/api/operator_license.go @@ -30,6 +30,9 @@ type License struct { // no longer be used in any capacity TerminationTime time.Time `json:"termination_time"` + // Whether the license will ignore termination + IgnoreTermination bool `json:"ignore_termination"` + // The product the license is valid for Product string `json:"product"` From 5755a75d8dae587b779bae1a8b44e4b5abc1cd63 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 12:45:34 -0800 Subject: [PATCH 041/421] Backport of Fix various flaky tests into release/1.15.x (#16406) * backport of commit 169a4aedee275ee0b9918a2484489a4ef90fddbc * backport of commit d7429dda9b85e33ff2b29d4d55708416c2ff6220 * backport of commit 7ee6f72dced7322f823f852e6742313188e69a3e --------- Co-authored-by: Chris S. Kim --- .../services/peerstream/stream_test.go | 4 +- .../services/peerstream/testing.go | 10 ++++ agent/xds/delta_test.go | 47 +++++++++++-------- agent/xds/xds_protocol_helpers_test.go | 4 +- command/debug/debug.go | 3 +- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index a0a4e9292ed7..ed5809e4554e 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -1252,8 +1252,8 @@ func TestStreamResources_Server_DisconnectsOnHeartbeatTimeout(t *testing.T) { }) testutil.RunStep(t, "stream is disconnected due to heartbeat timeout", func(t *testing.T) { - disconnectTime := ptr(it.FutureNow(1)) retry.Run(t, func(r *retry.R) { + disconnectTime := ptr(it.StaticNow()) status, ok := srv.StreamStatus(testPeerID) require.True(r, ok) require.False(r, status.Connected) @@ -1423,7 +1423,7 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient { }, })) - // Receive a services and roots subscription request pair from server + // Receive ExportedService, ExportedServiceList, and PeeringTrustBundle subscription requests from server receivedSub1, err := client.Recv() require.NoError(t, err) receivedSub2, err := client.Recv() diff --git a/agent/grpc-external/services/peerstream/testing.go b/agent/grpc-external/services/peerstream/testing.go index 4f0297a6c522..5eb575c06aa9 100644 --- a/agent/grpc-external/services/peerstream/testing.go +++ b/agent/grpc-external/services/peerstream/testing.go @@ -150,6 +150,16 @@ func (t *incrementalTime) Now() time.Time { return t.base.Add(dur) } +// StaticNow returns the current internal clock without advancing it. +func (t *incrementalTime) StaticNow() time.Time { + t.mu.Lock() + defer t.mu.Unlock() + + dur := time.Duration(t.next) * time.Second + + return t.base.Add(dur) +} + // FutureNow will return a given future value of the Now() function. // The numerical argument indicates which future Now value you wanted. The // value must be > 0. diff --git a/agent/xds/delta_test.go b/agent/xds/delta_test.go index 23d60198bbf8..a61050f8457d 100644 --- a/agent/xds/delta_test.go +++ b/agent/xds/delta_test.go @@ -10,7 +10,6 @@ import ( "github.com/armon/go-metrics" envoy_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" rpcstatus "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" @@ -21,8 +20,10 @@ import ( "github.com/hashicorp/consul/agent/grpc-external/limiter" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/version" ) @@ -1057,19 +1058,23 @@ func TestServer_DeltaAggregatedResources_v3_ACLEnforcement(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // aclResolve may be called in a goroutine even after a + // testcase tt returns. Capture the variable as tc so the + // values don't swap in the next iteration. + tc := tt aclResolve := func(id string) (acl.Authorizer, error) { - if !tt.defaultDeny { + if !tc.defaultDeny { // Allow all return acl.RootAuthorizer("allow"), nil } - if tt.acl == "" { + if tc.acl == "" { // No token and defaultDeny is denied return acl.RootAuthorizer("deny"), nil } // Ensure the correct token was passed - require.Equal(t, tt.token, id) + require.Equal(t, tc.token, id) // Parse the ACL and enforce it - policy, err := acl.NewPolicyFromSource(tt.acl, nil, nil) + policy, err := acl.NewPolicyFromSource(tc.acl, nil, nil) require.NoError(t, err) return acl.NewPolicyAuthorizerWithDefaults(acl.RootAuthorizer("deny"), []*acl.Policy{policy}, nil) } @@ -1095,13 +1100,15 @@ func TestServer_DeltaAggregatedResources_v3_ACLEnforcement(t *testing.T) { // If there is no token, check that we increment the gauge if tt.token == "" { - data := scenario.sink.Data() - require.Len(t, data, 1) - - item := data[0] - val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"] - require.True(t, ok) - require.Equal(t, float32(1), val.Value) + retry.Run(t, func(r *retry.R) { + data := scenario.sink.Data() + require.Len(r, data, 1) + + item := data[0] + val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"] + require.True(r, ok) + require.Equal(r, float32(1), val.Value) + }) } if !tt.wantDenied { @@ -1138,13 +1145,15 @@ func TestServer_DeltaAggregatedResources_v3_ACLEnforcement(t *testing.T) { // If there is no token, check that we decrement the gauge if tt.token == "" { - data := scenario.sink.Data() - require.Len(t, data, 1) - - item := data[0] - val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"] - require.True(t, ok) - require.Equal(t, float32(0), val.Value) + retry.Run(t, func(r *retry.R) { + data := scenario.sink.Data() + require.Len(r, data, 1) + + item := data[0] + val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"] + require.True(r, ok) + require.Equal(r, float32(0), val.Value) + }) } }) } diff --git a/agent/xds/xds_protocol_helpers_test.go b/agent/xds/xds_protocol_helpers_test.go index 8c4481515c8b..2edd05b9fb20 100644 --- a/agent/xds/xds_protocol_helpers_test.go +++ b/agent/xds/xds_protocol_helpers_test.go @@ -166,9 +166,6 @@ func newTestServerDeltaScenario( ) *testServerScenario { mgr := newTestManager(t) envoy := NewTestEnvoy(t, proxyID, token) - t.Cleanup(func() { - envoy.Close() - }) sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) cfg := metrics.DefaultConfig("consul.xds.test") @@ -177,6 +174,7 @@ func newTestServerDeltaScenario( metrics.NewGlobal(cfg, sink) t.Cleanup(func() { + envoy.Close() sink := &metrics.BlackholeSink{} metrics.NewGlobal(cfg, sink) }) diff --git a/command/debug/debug.go b/command/debug/debug.go index 017f42b77a2c..dd03286d68fc 100644 --- a/command/debug/debug.go +++ b/command/debug/debug.go @@ -270,7 +270,8 @@ func (c *cmd) prepare() (version string, err error) { // If none are specified we will collect information from // all by default if len(c.capture) == 0 { - c.capture = defaultTargets + c.capture = make([]string, len(defaultTargets)) + copy(c.capture, defaultTargets) } // If EnableDebug is not true, skip collecting pprof From 407f112a2f27e6f58962ed0207f755feabcc6fa0 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 23 Feb 2023 12:55:27 -0800 Subject: [PATCH 042/421] backport of commit ad47f9be23ab6912707f48cd77b058232dcc0ec2 (#16399) Co-authored-by: Nathan Coleman --- website/content/api-docs/operator/usage.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/api-docs/operator/usage.mdx b/website/content/api-docs/operator/usage.mdx index ae1e9d87ccb8..75298e32b209 100644 --- a/website/content/api-docs/operator/usage.mdx +++ b/website/content/api-docs/operator/usage.mdx @@ -164,4 +164,4 @@ $ curl \ - `PartitionNamespaceConnectServiceInstances` is the total number of Connect service instances registered in the datacenter, - by partition and namespace. \ No newline at end of file + by partition and namespace. From d8b6aaee0740e4c09e0152808fde814ab769d91a Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Thu, 23 Feb 2023 16:55:00 -0500 Subject: [PATCH 043/421] Docs/cluster peering 1.15 updates (#16291) (#16410) * initial commit * initial commit * Overview updates * Overview page improvements * More Overview improvements * improvements * Small fixes/updates * Updates * Overview updates * Nav data * More nav updates * Fix * updates * Updates + tip test * Directory test * refining * Create restructure w/ k8s * Single usage page * Technical Specification * k8s pages * typo * L7 traffic management * Manage connections * k8s page fix * Create page tab corrections * link to k8s * intentions * corrections * Add-on intention descriptions * adjustments * Missing * Diagram improvements * Final diagram update * Apply suggestions from code review * diagram name fix * Fixes * Updates to index.mdx * Tech specs page corrections * Tech specs page rename * update link to tech specs * K8s - new pages + tech specs * k8s - manage peering connections * k8s L7 traffic management * Separated establish connection pages * Directory fixes * Usage clean up * k8s docs edits * Updated nav data * CodeBlock Component fix * filename * CodeBlockConfig removal * Redirects * Update k8s filenames * Reshuffle k8s tech specs for clarity, fmt yaml files * Update general cluster peering docs, reorder CLI > API > UI, cross link to kubernetes * Fix config rendering in k8s usage docs, cross link to general usage from k8s docs * fix legacy link * update k8s docs * fix nested list rendering * redirect fix * page error --------- Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu Co-authored-by: Tu Nguyen --- .../connect/cluster-peering/configuration.mdx | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 website/content/docs/connect/cluster-peering/configuration.mdx diff --git a/website/content/docs/connect/cluster-peering/configuration.mdx b/website/content/docs/connect/cluster-peering/configuration.mdx deleted file mode 100644 index 53fab696c0c2..000000000000 --- a/website/content/docs/connect/cluster-peering/configuration.mdx +++ /dev/null @@ -1,56 +0,0 @@ ---- -layout: docs -page_title: Cluster Peering Configuration -description: >- - ---- - -# Enabling Service-to-service Traffic Across Peered Clusters - -The topic provides an overview of the configuration options and process for cluster peering. - -## Prerequisites - -To configure mesh gateways for cluster peering, make sure your Consul environment meets the following requirements: - -- Consul version 1.14.0 or newer. -- A local Consul agent is required to manage mesh gateway configuration. -- Use [Envoy proxies](/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. - -## Configuration - -Configure the following settings to register and use the mesh gateway as a service in Consul. - -### Gateway registration - -- Specify `mesh-gateway` in the `kind` field to register the gateway with Consul. -- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For Envoy, refer to the [Gateway Options](/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional configuration information. - -Alternatively, you can also use the CLI to spin up and register a gateway in Consul. For additional information, refer to the [`consul connect envoy` command](/commands/connect/envoy#mesh-gateways). - -### Sidecar registration - -- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams` documentation](/docs/connect/registration/service-registration#upstream-configuration-reference) for details. -- The service `proxy.upstreams.destination_name` is always required. -- The `proxy.upstreams.destination_peer` must be configured to enable cross-cluster traffic. -- The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. - -### Service exports - -- Include the `exported-services` configuration entry to enable Consul to export services contained in a cluster to one or more additional clusters. For additional information, refer to the [Exported Services documentation](/docs/connect/config-entries/exported-services). - -### ACL configuration - -If ACLs are enabled, you must add a token granting `service:write` for the gateway's service name and `service:read` for all services in the Enterprise admin partition or OSS datacenter to the gateway's service definition. - -These permissions authorize the token to route communications for other Consul service mesh services. - -You must also grant `mesh:write` to mesh gateways routing peering traffic in the data plane. - -This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. - -### Modes - -Modes are configurable as either `remote` or `local` for mesh gateways that connect peered clusters. -The `none` setting is invalid for mesh gateways in peered clusters and will be ignored by the gateway. -By default, all proxies connecting to peered clusters use mesh gateways in [remote mode](/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). \ No newline at end of file From a5bf79ef40714d755d0fc8508c16e588c8b5f5a1 Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Thu, 23 Feb 2023 17:18:16 -0500 Subject: [PATCH 044/421] Backport of Native API Gateway Docs (#16365) (#16409) * Create empty files * Copy over content for overview * Copy over content for usage * Copy over content for api-gateway config * Copy over content for http-route config * Copy over content for tcp-route config * Copy over content for inline-certificate config * Add docs to the sidebar * Clean up overview. Start cleaning up usage * Add BETA badge to API Gateways portion of nav * Fix header * Fix up usage * Fix up API Gateway config * Update paths to be consistent w/ other gateway docs * Fix up http-route * Fix up inline-certificate * rename path * Fix up tcp-route * Add CodeTabs * Add headers to config pages * Fix configuration model for http route and inline certificate * Add version callout to API gateway overview page * Fix values for inline certificate * Fix values for api gateway configuration * Fix values for TCP Route config * Fix values for HTTP Route config * Adds link from k8s gateway to vm gateway page * Remove versioning warning * Serve overview page at ../api-gateway, consistent w/ mesh-gateway * Remove weight field from tcp-route docs * Linking to usage instead of overview from k8s api-gateway to vm api-gateway * Fix issues in usage page * Fix links in usage * Capitalize Kubernetes * Apply suggestions from code review * remove optional callout * Apply suggestions from code review * Apply suggestions from code review * Apply suggestions from code review * Update website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx * Fix formatting of Hostnames * Update website/content/docs/api-gateway/index.mdx * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx * Add cross-linking of config entries * Fix rendering error on new operator usage docs * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx * Apply suggestions from code review * Apply suggestions from code review * Add BETA badges to config entry links * http route updates * Add Enterprise keys * Use map instead of list for meta field, use consistent formatting * Convert spaces to tabs * Add all Enterprise info to TCP Route * Use pascal case for JSON api-gateway example * Add enterprise to HCL api-gw cfg * Use pascal case for missed JSON config fields * Add enterprise to JSON api-gw cfg * Add enterprise to api-gw values * adds enterprise to http route * Update website/content/docs/connect/gateways/api-gateway/index.mdx * Add enterprise to api-gw spec * Add missing namespace, partition + meta to specification * fixes for http route * Fix ordering of API Gatetway cfg spec items * whitespace * Add linking of values to tcp * Apply suggestions from code review * Fix comma in wrong place * Apply suggestions from code review * Move Certificates down * Apply suggestions from code review * Tabs to spaces in httproute * Use configuration entry instead of config entry * Fix indentations on api-gateway and tcp-route * Add whitespace between code block and prose * Apply suggestions from code review * adds <> to http route --------- Co-authored-by: Thomas Eckert Co-authored-by: Melisa Griffin Co-authored-by: Tu Nguyen Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: Melisa Griffin Co-authored-by: Andrew Stucki Co-authored-by: danielehc <40759828+danielehc@users.noreply.github.com> Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --- website/content/docs/api-gateway/index.mdx | 4 +- .../api-gateway/configuration/api-gateway.mdx | 331 +++++++++ .../api-gateway/configuration/http-route.mdx | 678 ++++++++++++++++++ .../configuration/inline-certificate.mdx | 127 ++++ .../api-gateway/configuration/tcp-route.mdx | 256 +++++++ .../connect/gateways/api-gateway/index.mdx | 28 + .../connect/gateways/api-gateway/usage.mdx | 211 ++++++ website/data/docs-nav-data.json | 76 +- 8 files changed, 1708 insertions(+), 3 deletions(-) create mode 100644 website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx create mode 100644 website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx create mode 100644 website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx create mode 100644 website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx create mode 100644 website/content/docs/connect/gateways/api-gateway/index.mdx create mode 100644 website/content/docs/connect/gateways/api-gateway/usage.mdx diff --git a/website/content/docs/api-gateway/index.mdx b/website/content/docs/api-gateway/index.mdx index 462c286e5084..83372546ac46 100644 --- a/website/content/docs/api-gateway/index.mdx +++ b/website/content/docs/api-gateway/index.mdx @@ -5,9 +5,9 @@ description: >- Consul API Gateway enables external network client access to a service mesh on Kubernetes and forwards requests based on path or header information. Learn about how the k8s Gateway API specification configures Consul API Gateway so you can control access and simplify traffic management. --- -# API Gateway for Kubernetes Overview +# API Gateway for Kubernetes overview -This topic provides an overview of the Consul API Gateway. +This topic provides an overview of the Consul API Gateway for deploying on Kubernetes. If you would like to deploy on virtual machines, refer to [API Gateways on Virtual Machines](/consul/docs/connect/gateways/api-gateway/usage). ## What is Consul API Gateway? diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx new file mode 100644 index 000000000000..11ad78f2d276 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx @@ -0,0 +1,331 @@ +--- +layout: docs +page_title: API Gateway Configuration Entry Reference +description: Learn how to configure a Consul API Gateway on VMs. +--- + +# API gateway configuration entry reference + +This topic provides reference information for the API gateway configuration entry that you can deploy to networks in virtual machine (VM) environments. For reference information about configuring Consul API gateways on Kubernetes, refer to [Gateway Resource Configuration](/consul/docs/api-gateway/configuration/gateway). + +## Introduction + +A gateway is a type of network infrastructure that determines how service traffic should be handled. Gateways contain one or more listeners that bind to a set of hosts and ports. An HTTP Route or TCP Route can then attach to a gateway listener to direct traffic from the gateway to a service. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in an `api-gateway` configuration entry. Click on a property name to view additional details, including default values. + +- [`Kind`](#kind): string | must be `"api-gateway"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Listeners`](#listeners): list of objects | no default + - [`Name`](#listeners-name): string | no default + - [`Port`](#listeners-port): number | no default + - [`Hostname`](#listeners-hostname): string | `"*"` + - [`Protocol`](#listeners-protocol): string | `"tcp"` + - [`TLS`](#listeners-tls): map | none + - [`MinVersion`](#listeners-tls-minversion): string | no default + - [`MaxVersion`](#listeners-tls-maxversion): string | no default + - [`CipherSuites`](#listeners-tls-ciphersuites): list of strings | Envoy default cipher suites + - [`Certificates`](#listeners-tls-certificates): list of objects | no default + - [`Kind`](#listeners-tls-certificates-kind): string | must be `"inline-certificate"` + - [`Name`](#listeners-tls-certificates-name): string | no default + - [`Namespace`](#listeners-tls-certificates-namespace): string | no default + - [`Partition`](#listeners-tls-certificates-partition): string | no default + +## Complete configuration + +When every field is defined, an `api-gateway` configuration entry has the following form: + + + +```hcl +Kind = "api-gateway" +Name = "" +Namespace = "" +Partition = "" + +Meta = { + = "" +} + +Listeners = [ + { + Port = + Name = "" + Protocol = "" + TLS = { + MaxVersion = "" + MinVersion = "" + CipherSuites = [ + "" + ] + Certificates = [ + { + Kind = "inline-certificate" + Name = "" + Namespace = "" + Partition = "" + } + ] + } + } +] +``` + +```json +{ + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Listeners": [ + { + "Name": "", + "Port": , + "Protocol": "", + "TLS": { + "MaxVersion": "", + "MinVersion": "", + "CipherSuites": [ + "" + ], + "Certificates": [ + { + "Kind": "inline-certificate", + "Name": "", + "Namespace": "", + "Partition": "" + } + ] + } + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the +`api-gateway` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. This must be +`api-gateway`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to `"api-gateway"`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Listeners[]` + +Specifies a list of listeners that gateway should set up. Listeners are +uniquely identified by their port number. + +#### Values + +- Default: none +- This field is required. +- Data type: List of maps. Each member of the list contains the following fields: + - [`Name`](#listeners-name) + - [`Port`](#listeners-port) + - [`Hostname`](#listeners-hostname) + - [`Protocol`](#listeners-protocol) + - [`TLS`](#listeners-tls) + +### `Listeners[].Name` + +Specifies the unique name for the listener. This field accepts letters, numbers, and hyphens. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Listeners[].Port` + +Specifies the port number that the listener receives traffic on. + +#### Values + +- Default: `0` +- This field is required. +- Data type: integer + +### `Listeners[].Hostname` + +Specifies the hostname that the listener receives traffic on. + +#### Values + +- Default: `"*"` +- This field is optional. +- Data type: string + +### `Listeners[].Protocol` + +Specifies the protocol associated with the listener. + +#### Values + +- Default: none +- This field is required. +- The data type is one of the following string values: `"tcp"` or `"http"`. + +### `Listeners[].TLS` + +Specifies the TLS configurations for the listener. + +#### Values + +- Default: none +- Map that contains the following fields: + - [`MaxVersion`](#listeners-tls-maxversion) + - [`MinVersion`](#listeners-tls-minversion) + - [`CipherSuites`](#listeners-tls-ciphersuites) + - [`Certificates`](#listeners-tls-certificates) + +### `Listeners[].TLS.MaxVersion` + +Specifies the maximum TLS version supported for the listener. + +#### Values + +- Default depends on the version of Envoy: + - Envoy 1.22.0 and later default to `TLSv1_2` + - Older versions of Envoy default to `TLSv1_0` +- Data type is one of the following string values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.MinVersion` + +Specifies the minimum TLS version supported for the listener. + +#### Values + +- Default: none +- Data type is one of the following string values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.CipherSuites[]` + +Specifies a list of cipher suites that the listener supports when negotiating connections using TLS 1.2 or older. + +#### Values + +- Defaults to the ciphers supported by the version of Envoy in use. Refer to the + [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tlsparameters-cipher-suites) + for details. +- Data type: List of string values. Refer to the + [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) + for a list of supported ciphers. + +### `Listeners[].TLS.Certificates[]` + +The list of references to inline certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- Data type: List of maps. Each member of the list has the following fields: + - [`Kind`](#listeners-tls-certificates-kind) + - [`Name`](#listeners-tls-certificates-name) + - [`Namespace`](#listeners-tls-certificates-namespace) + - [`Partition`](#listeners-tls-certificates-partition) + +### `Listeners[].TLS.Certificates[].Kind` + +The list of references to inline-certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- This field is required and must be set to `"inline-certificate"`. +- Data type: string + +### `Listeners[].TLS.Certificates[].Name` + +The list of references to inline certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- This field is required. +- Data type: string + +### `Listeners[].TLS.Certificates[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the certificate can be found. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Listeners[].TLS.Certificates[].Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the certificate can be found. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx new file mode 100644 index 000000000000..c492e331e2ae --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx @@ -0,0 +1,678 @@ +--- +layout: docs +page_title: HTTP Route Configuration +description: Learn how to configure an HTTP Route bound to an API Gateway on VMs. +--- + +# HTTP route configuration reference + +This topic provides reference information for the gateway routes configuration entry. Refer to [Route Resource Configuration](/consul/docs/api-gateway/configuration/routes) for information about configuring API gateway routes in Kubernetes environments. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `http-route` configuration entry. Click on a property name +to view additional details, including default values. + +- [`Kind`](#kind): string | must be `http-route` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Hostnames`](#hostnames): list | no default +- [`Parents`](#parents): list | no default + - [`Kind`](#parents-kind): string | must be `api-gateway` + - [`Name`](#parents-name): string | no default + - [`Namespace`](#parents-namespace): string | no default + - [`Partition`](#parents-partition): string | no default + - [`SectionName`](#parents-sectionname): string | no default +- [`Rules`](#rules): list | no default + - [`Filters`](#rules-filters): map | no default + - [`Headers`](#rules-filters-headers): list | no default + - [`Add`](#rules-filters-headers-add): map | no default + - [`Remove`](#rules-filters-headers-remove): list | no default + - [`Set`](#rules-filters-headers-set): map | no default + - [`URLRewrite`](#rules-filters-urlrewrite): map | no default + - [`Path`](#rules-filters-urlrewrite-path): string | no default + - [`Matches`](#rules-matches): list | no default + - [`Headers`](#rules-matches-headers): list | no default + - [`Match`](#rules-matches-headers-match): string | no default + - [`Name`](#rules-matches-headers-name): string | no default + - [`Value`](#rules-matches-headers-value): string | no default + - [`Method`](#rules-matches-method): string | no default + - [`Path`](#rules-matches-path): map | no default + - [`Match`](#rules-matches-path-match): string | no default + - [`Value`](#rules-matches-path-value): string | no default + - [`Query`](#rules-matches-query): list | no default + - [`Match`](#rules-matches-query-match): string | no default + - [`Name`](#rules-matches-query-name): string | no default + - [`Value`](#rules-matches-query-value): string | no default + - [`Services`](#rules-services): list | no default + - [`Name`](#rules-services-name): string | no default + - [`Namespace`](#rules-services-namespace): string + - [`Partition`](#rules-services-partition): string + - [`Weight`](#rules-services-weight): number | `1` + - [`Filters`](#rules-services-filters): map | no default + - [`Headers`](#rules-services-filters-headers): list | no default + - [`Add`](#rules-services-filters-headers-add): map | no default + - [`Remove`](#rules-services-filters-headers-remove): list | no default + - [`Set`](#rules-services-filters-headers-set): map | no default + - [`URLRewrite`](#rules-services-filters-urlrewrite): map | no default + - [`Path`](#rules-services-filters-urlrewrite-path): string | no default + +## Complete configuration + +When every field is defined, an `http-route` configuration entry has the following form: + + + +```hcl +Kind = "http-route" +Name = "" +Namespace = "" +Partition = "" +Meta = { + "" = "" +} +Hostnames = [""] + +Parents = [ + { + Kind = "api-gateway" + Name = "" + Namespace = "" + Partition = "" + SectionName = "" + } +] + +Rules = [ + { + Filters = { + Headers = [ + { + Add = { + "" = "" + } + Remove = [ + "" + ] + Set = { + "" = "" + } + } + ] + URLRewrite = { + Path = "" + } + } + Matches = [ + { + Headers = [ + { + Match = "" + Name = "" + Value = "" + } + ] + Method = "" + Path = { + Match = "" + Value = "" + } + Query = [ + { + Match = "" + Name = "" + Value = "" + } + ] + } + ] + Services = [ + { + Name = "" + Namespace = "" + Partition = "" + Weight = "" + Filters = { + Headers = [ + { + Add = { + "" = "" + } + Remove = [ + "" + ] + Set = { + "" = "" + } + } + ] + URLRewrite = { + Path = "" + } + } + } + ] + } +] + +``` + +```json +{ + "Kind": "http-route", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Hostnames": [ + "" + ], + "Parents": [ + { + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "SectionName": "" + } + ], + "Rules": [ + { + "Filters": [ + { + "Headers": [ + { + "Add": [ + { + "": "" + } + ], + "Remove": ["

"], + "Set": [ + { + "": "" + } + ] + } + ], + "URLRewrite": [ + { + "Path": "" + } + ] + } + ], + "Matches": [ + { + "Headers": [ + { + "Match": "", + "Name": "", + "Value": "" + } + ], + "Method": "", + "Path": [ + { + "Match": "", + "Value": "" + } + ], + "Query": [ + { + "Match": "", + "Name": "", + "Value": "" + } + ] + } + ], + "Services": [ + { + "Name": "", + "Namespace": "", + "Partition": "", + "Weight": "", + "Filters": [ + { + "Headers": [ + { + "Add": [ + { + "" + } + ], + "Remove": ["
"], + "Set": [ + { + "" + } + ] + } + ], + "URLRewrite": [ + { + "Path": "" + } + ] + } + ] + } + ] + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the `http-route` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. For HTTP routes, this must be `http-route`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to `"http-route"`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the route. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Parents[]` + +Specifies the list of gateways that this route binds to. + +#### Values + +- Default: none +- Data type: List of map. Each member of the list contains the following fields: + - `Kind` + - `Name` + - `Namespace` + - `Partition` + - `SectionName` + +### `Parents[].Kind` + +Specifies the type of resource to bind to. This field is required and must be +set to `"api-gateway"` + +#### Values + +- Default: none +- This field is required. +- Data type: string value set to `"api-gateway"` + +### `Parents[].Name` + +Specifies the name of the api-gateway to bind to. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents[].Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents[].SectionName` + +Specifies the name of the listener to bind to on the `api-gateway`. If left +empty, this route binds to _all listeners_ on the parent gateway. + +#### Values + +- Default: "" +- Data type: string + +### `Rules[]` + +Specifies the list of HTTP-based routing rules that this route uses to construct a route table. + +#### Values + +- Default: +- Data type: List of maps. Each member of the list contains the following fields: + - `Filters` + - `Matches` + - `Services` + +### `Rules[].Filters` + +Specifies the list of HTTP-based filters used to modify a request prior to routing it to the upstream service. + +#### Values + +- Default: none +- Data type: Map that contains the following fields: + - `Headers` + - `UrlRewrite` + +### `Rules[].Filters.Headers[]` + +Defines operations to perform on matching request headers when an incoming request matches the `Rules.Matches` configuration. + +#### Values + +This field contains the following configuration objects: + +| Parameter | Description | Type | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `set` | Configure this field to rewrite the HTTP request header. It specifies the name of an HTTP header to overwrite and the new value to set. Any existing values associated with the header name are overwritten. You can specify the following configurations:
  • `name`: Required string that specifies the name of the HTTP header to set.
  • `value`: Required string that specifies the value of the HTTP header to set.
| List of maps | +| `add` | Configure this field to append the request header with a new value. It specifies the name of an HTTP header to append and the values to add. You can specify the following configurations:
  • `name`: Required string that specifies the name of the HTTP header to append.
  • `value`: Required string that specifies the value of the HTTP header to add.
| List of maps | +| `remove` | Configure this field to specify an array of header names to remove from the request header. | List of strings | + +### `Rules[].Filters.URLRewrite` + +Specifies rule for rewriting the URL of incoming requests when an incoming request matches the `Rules.Matches` configuration. + +#### Values + +- Default: none +- This field is a map that contains a `Path` field. + +### Rules[].Filters.URLRewrite.Path + +Specifies a path that determines how Consul API Gateway rewrites a URL path. Refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information. + +#### Values + +The following table describes the parameters for `path`: + +| Parameter | Description | Type | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------ | +| `replacePrefixMatch` | Specifies a value that replaces the path prefix for incoming HTTP requests. The operation only affects the path prefix. The rest of the path is unchanged. | String | +| `type` | Specifies the type of replacement to use for the URL path. You can specify the following values:
  • `ReplacePrefixMatch`: Replaces the matched prefix of the URL path (default).
| String | + +### `Rules[].Matches[]` + +Specifies the matching criteria used in the routing table. When an incoming +request matches the given HTTPMatch configuration, traffic routes to +services specified in the [`Rules.Services`](#rules-services) field. + +#### Values + +- Default: none +- Data type: List containing maps. Each member of the list contains the following fields: + - `Headers` + - `Method` + - `Path` + - `Query` + +### `Rules[].Matches[].Headers[]` + +Specifies rules for matching incoming request headers. You can specify multiple rules in a list, as well as multiple lists of rules. If all rules in a single list are satisfied, then the route forwards the request to the appropriate service defined in the [`Rules.Services`](#rules-services) configuration. You can create multiple `Header[]` lists to create a range of matching criteria. When at least one list of matching rules are satisfied, the route forwards the request to the appropriate service defined in the [`Rules.Services`](#rules-services) configuration. + +#### Values + +- Default: none +- Data type: List containing maps with the following fields: + - `Match` + - `Name` + - `Value` + +### `Rules.Matches.Headers.Match` + +Specifies type of match for headers: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules.Matches.Headers.Name` + +Specifies the name of the header to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches.Headers.Value` + +Specifies the value of the header to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Method` + +Specifies a list of strings that define matches based on HTTP request method. + +#### Values + +Specify one of the following string values: + +- `HEAD` +- `POST` +- `PUT` +- `PATCH` +- `GET` +- `DELETE` +- `OPTIONS` +- `TRACE` +- `CONNECT` + +### `Rules[].Matches[].Path` + +Specifies the HTTP method to match. + +#### Values + +- Default: none +- Data type: map containing the following fields: + - `Match` + - `Value` + +### `Rules[].Matches[].Path.Match` + +Specifies type of match for the path: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Path.Value` + +Specifies the value of the path to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[]` + +Specifies how a match is completed on a request’s query parameters. + +#### Values + +- Default: none +- Data type: List of map that contains the following fields: + - `Match` + - `Name` + - `Value` + +### `Rules[].Matches[].Query[].Match` + +Specifies type of match for query parameters: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[].Name` + +Specifies the name of the query parameter to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[].Value` + +Specifies the value of the query parameter to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Services[]` + +Specifies the service that the API gateway routes incoming requests to when the +requests match the `Rules.Matches` configuration. + +#### Values + +- Default: none +- This field contains a list of maps. Each member of the list contains the following fields: + - `Name` + - `Weight` + - `Filters` + - `Namespace` + - `Partition` + +### `Rules[].Services[].Name` + +Specifies the name of an HTTP-based service to route to. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Services[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Rules[].Services.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Rules[].Services[].Weight` + +Specifies the proportion of requests forwarded to the specified service. The +proportion is determined by dividing the value of the weight by the sum of all +weights in the service list. For non-zero values, there may be some deviation +from the exact proportion depending on the precision an implementation +supports. Weight is not a percentage and the sum of weights does not need to +equal 100. + +#### Values + +- Default: none +- Data type: integer + +### `Rules[].Services[].Filters` + +Specifies the list of HTTP-based filters used to modify a request prior to +routing it to this upstream service. + +#### Values + +- Default: none +- Data type: Map that contains the following fields: + - `Headers` + - `UrlRewrite` + +### `Rules[].Services[].Filters.Headers[]` + +Defines operations to perform on matching request headers. + +#### Values + +This field contains the following configuration objects: + +| Parameter | Description | Type | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `set` | Configure this field to rewrite the HTTP request header. It specifies the name of an HTTP header to overwrite and the new value to set. Any existing values associated with the header name are overwritten. You can specify the following configurations:
  • `name`: Required string that specifies the name of the HTTP header to set.
  • `value`: Required string that specifies the value of the HTTP header to set.
| List of maps | +| `add` | Configure this field to append the request header with a new value. It specifies the name of an HTTP header to append and the values to add. You can specify the following configurations:
  • `name`: Required string that specifies the name of the HTTP header to append.
  • `value`: Required string that specifies the value of the HTTP header to add.
| List of maps | +| `remove` | Configure this field to specify an array of header names to remove from the request header. | List of strings | + +### `Rules[].Services[].Filters.URLRewrite` + +Specifies rule for rewriting the URL of incoming requests. + +#### Values + +- Default: none +- This field is a map that contains a `Path` field. diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx new file mode 100644 index 000000000000..4fca7c54ee60 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx @@ -0,0 +1,127 @@ +--- +layout: docs +page_title: Inline Certificate Configuration Reference +description: Learn how to configure an inline certificate bound to an API Gateway on VMs. +--- + +# Inline certificate configuration reference + +This topic provides reference information for the gateway inline certificate +configuration entry. For information about certificate configuration for Kubernetes environments, refer to [Gateway Resource Configuration](/consul/docs/api-gateway/configuration/gateway). + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `inline-certificate` configuration entry. Click on a property name +to view additional details, including default values. + +- [`Kind`](#kind): string | must be `"inline-certificate"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Certificate`](#certificate): string | no default +- [`PrivateKey`](#privatekey): string | no default + +## Complete configuration + +When every field is defined, an `inline-certificate` configuration entry has the following form: + + + +```HCL +Kind = "inline-certificate" +Name = "" + +Meta = { + "" = "" +} + +Certificate = "" +PrivateKey = "" +``` + +```JSON +{ + "Kind": "inline-certificate", + "Name": "", + "Meta": { + "any key": "any value" + } + "Certificate": "", + "PrivateKey": "" +} +``` + + + +## Specification + +### `Kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: string that must equal `"inline-certificate"` + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, such +as applying a configuration entry to a specific cluster. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Certificate` + +Specifies the inline public certificate to use for TLS. + +#### Values + +- Default: none +- This field is required. +- Data type: string value of the public certificate + +### `PrivateKey` + +Specifies the inline private key to use for TLS. + +#### Values + +- Default: none +- This field is required. +- Data type: string value of the private key diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx new file mode 100644 index 000000000000..23b32662d923 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx @@ -0,0 +1,256 @@ +--- +layout: docs +page_title: TCP Route Configuration Reference +description: Learn how to configure a TCP Route that is bound to an API gateway on VMs. +--- + +# TCP route configuration Reference + +This topic provides reference information for the gateway TCP routes configuration +entry. Refer to [Route Resource Configuration](/consul/docs/api-gateway/configuration/routes) for information +about configuring API gateways in Kubernetes environments. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `tcp-route` configuration entry. Click on a property name to +view additional details, including default values. + +- [`Kind`](#kind): string | must be `"tcp-route"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Services`](#services): list | no default + - [`Name`](#services-name): string | no default + - [`Namespace`](#services-namespace): string | no default + - [`Partition`](#services-partition): string | no default +- [`Parents`](#parents): list | no default + - [`Kind`](#parents-kind): string | must be `"api-gateway"` + - [`Name`](#parents-name): string | no default + - [`Namespace`](#parents-namespace): string | no default + - [`Partition`](#parents-partition): string | no default + - [`SectionName`](#parents-sectionname): string | no default + +## Complete configuration + +When every field is defined, a `tcp-route` configuration entry has the following form: + + + +```HCL +Kind = "tcp-route" +Name = "" +Namespace = "" +Partition = "" + +Meta = { + "" = "" +} + +Services = [ + { + Name = "" + Namespace = "" + Partition = "" + } +] + + +Parents = [ + { + Kind = "api-gateway" + Name = "" + Namespace = "" + Partition = "" + SectionName = "" + } +] +``` + +```JSON +{ + "Kind": "tcp-route", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Services": [ + { + "Name": "" + "Namespace": "", + "Partition": "", + } + ], + "Parents": [ + { + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "SectionName": "" + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the +`tcp-route` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. This must be set to +`"tcp-route"`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to`tcp-route`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Services` + +Specifies a TCP-based service the API gateway routes incoming requests +to. You can only specify one service. + +#### Values + +- Default: none +- The data type is a list of maps. Each member of the list contains the following fields: + - [`Name`](#services-name) + - [`Namespace`](#services-namespace) + - [`Partition`](#services-partition) + +### `Services.Name` + +Specifies the list of TCP-based services to route to. You can specify a maximum of one service. + +#### Values + +- Default: none +- Data type: string + +### `Services.Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the service is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Services.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the service is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents` + +Specifies the list of gateways that the route is bound to. + +#### Values + +- Default: none +- Data type: List of map. Each member of the list contains the following fields: + - [`Kind`](#parents-kind) + - [`Name`](#parents-name) + - [`Namespace`](#parents-namespace) + - [`Partition`](#parents-partition) + - [`SectionName`](#parents-sectionname) + +### `Parents.Kind` + +Specifies the type of resource to bind to. This field is required and must be +set to `"api-gateway"` + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents.Name` + +Specifies the name of the API gateway to bind to. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents.Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the parent is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the parent is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents.SectionName` + +Specifies the name of the listener defined in the [`api-gateway` configuration](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway) that the route binds to. If the field is configured to an empty string, the route binds to all listeners on the parent gateway. + +#### Values + +- Default: `""` +- Data type: string diff --git a/website/content/docs/connect/gateways/api-gateway/index.mdx b/website/content/docs/connect/gateways/api-gateway/index.mdx new file mode 100644 index 000000000000..832fd943b2ee --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/index.mdx @@ -0,0 +1,28 @@ +--- +layout: docs +page_title: API Gateways Overview +description: API gateways are objects in Consul that enable ingress requests to services in your service mesh. Learn about API gateways for VMs in this overview. +--- + +# API gateway overview + +API gateways enable external network clients to access applications and services +running in a Consul datacenter. This type of network traffic is commonly +called _north-south_ network traffic because it refers to the flow of +data into and out of a specific environment. API gateways can also forward +requests from clients to specific destinations based on path or request +protocol. + +API gateways solve the following primary use cases: + +- **Control access at the point of entry**: Set the protocols of external connection + requests and secure inbound connections with TLS certificates from trusted + providers, such as Verisign and Let's Encrypt. +- **Simplify traffic management**: Load balance requests across services and route + traffic to the appropriate service by matching one or more criteria, such as + hostname, path, header presence or value, and HTTP method. + +Consul supports API +gateways for virtual machines and Kubernetes networks. Refer to the following documentation for next steps: +- [API Gateways on VMs](/consul/docs/connect/gateways/api-gateway/usage) +- [API Gateways for Kubernetes](/consul/docs/api-gateway). diff --git a/website/content/docs/connect/gateways/api-gateway/usage.mdx b/website/content/docs/connect/gateways/api-gateway/usage.mdx new file mode 100644 index 000000000000..a5fea6e8176c --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/usage.mdx @@ -0,0 +1,211 @@ +--- +layout: docs +page_title: API Gateways on Virtual Machines +description: Learn how to configure and Consul API gateways and gateway routes on virtual machines so that you can enable ingress requests to services in your service mesh in VM environments. +--- + +# API gateways on virtual machines + +This topic describes how to deploy Consul API gateways to networks that operate +in virtual machine (VM) environments. If you want to implement an API gateway +in a Kubernetes environment, refer to [API Gateway for Kubernetes](/consul/docs/api-gateway). + +## Introduction + +Consul API gateways provide a configurable ingress points for requests into a Consul network. Usethe following configuration entries to set up API gateways: + +- [API gateway](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway): Provides an endpoint for requests to enter the network. Define listeners that expose ports on the endpoint for ingress. +- [HTTP routes](/consul/docs/connect/gateways/api-gateway/configuration/http-route) and [TCP routes](/consul/docs/connect/gateways/api-gateway/configuration/tcp-route): The routes attach to listeners defined in the gateway and control how requests route to services in the network. +- [Inline certificates](/consul/docs/connect/gateways/api-gateway/configuration/inline-certificate): Makes TLS certificates available to gateways so that requests between the user and the gateway endpoint are encrypted. + +You can configure and reuse configuration entries separately. You can define and attach routes and inline certificates to multiple gateways. + +The following steps describe the general workflow for deploying a Consul API +gateway to a VM environment: + +1. Create an API gateway configuration entry. The configuration entry includes + listener configurations and references to TLS certificates. +1. Deploy the API gateway configuration entry to create the listeners. +1. Create and deploy routes to bind to the gateway. + +Refer to [API Gateway for Kubernetes](/consul/docs/api-gateway) for information +about using Consul API gateway on Kubernetes. + +## Requirements + +The following requirements must be satisfied to use API gateways on VMs: + +- Consul 1.15 or later +- A Consul cluster with service mesh enabled. Refer to [`connect`](/consul/docs/agent/config/config-files#connect) +- Network connectivity between the machine deploying the API Gateway and a + Consul cluster agent or server + +If ACLs are enabled, you must present a token with the following permissions to +configure Consul and deploy API gateways: + +- `mesh: read` +- `mesh: write` + +Refer [Mesh Rules](/consul/docs/security/acl/acl-rules#mesh-rules) for +additional information about configuring policies that enable you to interact +with Consul API gateway configurations. + +## Create the API gateway configuration + +Create an API gateway configuration that defines listeners and TLS certificates +in the mesh. In the following example, the API gateway specifies an HTTP +listener on port `8443` that routes can use to connect external traffic to +services in the mesh. + +```hcl +Kind = "api-gateway" +Name = "my-gateway" + +// Each listener configures a port which can be used to access the Consul cluster +Listeners = [ + { + Port = 8443 + Name = "my-http-listener" + Protocol = "http" + TLS = { + Certificates = [ + { + Kind = "inline-certificate" + Name = "my-certificate" + } + ] + } + } +] +``` + +Refer to [API Gateway Configuration Reference](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway) for +information about all configuration fields. + +Gateways and routes are eventually-consistent objects that provide feedback +about their current state through a series of status conditions. As a result, +you must manually check the route status to determine if the route +bound to the gateway successfully. + +## Deploy the API gateway + +Use the `consul config write` command to implement the API gateway +configuration entries. The following command applies the configuration entry +for the main gateway object: + +```shell-session +$ consul config write gateways.hcl +``` + +Run the following command to deploy an API gateway instance: + +```shell-session +$ consul connect envoy -gateway api -register -service my-api-gateway +``` + +The command directs Consul to configure Envoy as an API gateway. + +## Route requests + +Define route configurations and bind them to listeners configured on the +gateway so that Consul can route incoming requests to services in the mesh. +Create HTTP or TCP routes by setting the `Kind` parameter to `http-route` or +`tcp-route` and configuring rules that define request traffic flows. + +The following example routes requests from the listener on the API gateway at +port `8443` to services in Consul based on the path of the request. When an +incoming request starts at path `/`, Consul forwards 90 percent of the requests +to the `ui` service and 10 percent to `experimental-ui`. Consul also forwards +requests starting with `/api` to `api`. + +```hcl +Kind = "http-route" +Name = "my-http-route" + +// Rules define how requests will be routed +Rules = [ + // Send all requests to UI services with 10% going to the "experimental" UI + { + Matches = [ + { + Path = { + Match = "prefix" + Value = "/" + } + } + ] + Services = [ + { + Name = "ui" + Weight = 90 + }, + { + Name = "experimental-ui" + Weight = 10 + } + ] + }, + // Send all requests that start with the path `/api` to the API service + { + Matches = [ + { + Path = { + Match = "prefix" + Value = "/api" + } + } + ] + Services = [ + { + Name = "api" + } + ] + } +] + +Parents = [ + { + Kind = "api-gateway" + Name = "my-gateway" + SectionName = "my-http-listener" + } +] +``` + +Create this configuration by saving it to a file called `my-http-route.hcl` and using the command + +```shell-session +$ consul config write my-http-route.hcl +``` + +Refer to [HTTP Route Configuration Entry Reference](/consul/docs/connect/gateways/api-gateway/configuration/http-route) +and [TCP Route Configuration Entry Reference](/consul/docs/connect/gateways/api-gateway/configuration/tcp-route) for details about route configurations. + +## Add a TLS certificate + +Define an [`inline-certificate` configuration entry](/consul/docs/connect/gateways/api-gateway/configuration/inline-certificate) with a name matching the name in the [API gateway listener configuration](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway#listeners) to bind the certificate to that listener. The inline certificate configuration entry takes a public certificate and private key in plaintext. + +The following example defines a certificate named `my-certificate`. API gateway configurations that specify `inline-certificate` in the `Certificate.Kind` field and `my-certificate` in the `Certificate.Name` field are able to use the certificate. + +```hcl +Kind = "inline-certificate" +Name = "my-certificate" + +Certificate = < Date: Thu, 23 Feb 2023 17:53:52 -0800 Subject: [PATCH 045/421] backport of commit ec864cb31a96c621f1b96e9d5ada7f48f3b76ea8 (#16414) Co-authored-by: Claire --- .release/ci.hcl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.release/ci.hcl b/.release/ci.hcl index 084450dd4cd0..25f64e4c6b78 100644 --- a/.release/ci.hcl +++ b/.release/ci.hcl @@ -38,6 +38,41 @@ event "prepare" { } } +## These are promotion and post-publish events +## they should be added to the end of the file after the verify event stanza. + +event "trigger-staging" { +// This event is dispatched by the bob trigger-promotion command +// and is required - do not delete. +} + +event "promote-staging" { + depends = ["trigger-staging"] + action "promote-staging" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "promote-staging" + config = "release-metadata.hcl" + } + + notification { + on = "always" + } +} + +event "promote-staging-docker" { + depends = ["promote-staging"] + action "promote-staging-docker" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "promote-staging-docker" + } + + notification { + on = "always" + } +} + event "trigger-production" { // This event is dispatched by the bob trigger-promotion command // and is required - do not delete. From 992801033c9dfa9200fea01fc09e35b6ac57f0bf Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 24 Feb 2023 10:08:48 -0800 Subject: [PATCH 046/421] backport of commit 6aea18239f4f093c7cbf0874cd5e728bfc396e7d (#16420) Co-authored-by: Kyle Havlovitz --- .../usage/instances/usage_instances.go | 19 +++++--- .../usage/instances/usage_instances_test.go | 44 +++++++++++++++---- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/command/operator/usage/instances/usage_instances.go b/command/operator/usage/instances/usage_instances.go index df997022ae05..c1c94caa69ba 100644 --- a/command/operator/usage/instances/usage_instances.go +++ b/command/operator/usage/instances/usage_instances.go @@ -33,8 +33,10 @@ type cmd struct { func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) - c.flags.BoolVar(&c.onlyBillable, "billable", false, "Display only billable service info.") - c.flags.BoolVar(&c.onlyConnect, "connect", false, "Display only Connect service info.") + c.flags.BoolVar(&c.onlyBillable, "billable", false, "Display only billable service info. "+ + "Cannot be used with -connect.") + c.flags.BoolVar(&c.onlyConnect, "connect", false, "Display only Connect service info."+ + "Cannot be used with -billable.") c.flags.BoolVar(&c.allDatacenters, "all-datacenters", false, "Display service counts from "+ "all datacenters.") @@ -54,6 +56,11 @@ func (c *cmd) Run(args []string) int { return 1 } + if c.onlyBillable && c.onlyConnect { + c.UI.Error("Cannot specify both -billable and -connect flags") + return 1 + } + // Create and test the HTTP client client, err := c.http.APIClient() if err != nil { @@ -219,22 +226,22 @@ func (c *cmd) Help() string { const ( synopsis = "Display service instance usage information" help = ` -Usage: consul usage instances [options] +Usage: consul operator usage instances [options] Retrieves usage information about the number of services registered in a given datacenter. By default, the datacenter of the local agent is queried. To retrieve the service usage data: - $ consul usage instances + $ consul operator usage instances To show only billable service instance counts: - $ consul usage instances -billable + $ consul operator usage instances -billable To show only connect service instance counts: - $ consul usage instances -connect + $ consul operator usage instances -connect For a full list of options and examples, please see the Consul documentation. ` diff --git a/command/operator/usage/instances/usage_instances_test.go b/command/operator/usage/instances/usage_instances_test.go index 7aabf030e272..0f41b79aa0b8 100644 --- a/command/operator/usage/instances/usage_instances_test.go +++ b/command/operator/usage/instances/usage_instances_test.go @@ -1,6 +1,7 @@ package instances import ( + "errors" "testing" "github.com/hashicorp/consul/agent" @@ -36,15 +37,40 @@ func TestUsageInstancesCommand(t *testing.T) { t.Fatal(err) } - ui := cli.NewMockUi() - c := New(ui) - args := []string{ - "-http-addr=" + a.HTTPAddr(), + cases := []struct { + name string + extraArgs []string + output string + err error + }{ + { + name: "basic output", + output: "Billable Service Instances Total: 2", + }, + { + name: "billable and connect flags together are invalid", + extraArgs: []string{"-billable", "-connect"}, + err: errors.New("Cannot specify both -billable and -connect"), + }, } - code := c.Run(args) - if code != 0 { - t.Fatalf("bad exit code %d: %s", code, ui.ErrorWriter.String()) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ui := cli.NewMockUi() + c := New(ui) + args := []string{ + "-http-addr=" + a.HTTPAddr(), + } + args = append(args, tc.extraArgs...) + + code := c.Run(args) + if tc.err != nil { + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), tc.err.Error()) + } else { + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), tc.output) + } + }) } - output := ui.OutputWriter.String() - require.Contains(t, output, "Billable Service Instances Total: 2") } From bf8c4ac6208240fec7d928627d625774beebd138 Mon Sep 17 00:00:00 2001 From: Tu Nguyen Date: Fri, 24 Feb 2023 14:13:28 -0800 Subject: [PATCH 047/421] Fixes cluster peering docs (remove deprecated file) (#16424) * backport of commit e878d2d3e435a724e26789ab6fda84d009961495 * Docs/cluster peering 1.15 updates (#16291) * initial commit * initial commit * Overview updates * Overview page improvements * More Overview improvements * improvements * Small fixes/updates * Updates * Overview updates * Nav data * More nav updates * Fix * updates * Updates + tip test * Directory test * refining * Create restructure w/ k8s * Single usage page * Technical Specification * k8s pages * typo * L7 traffic management * Manage connections * k8s page fix * Create page tab corrections * link to k8s * intentions * corrections * Add-on intention descriptions * adjustments * Missing * Diagram improvements * Final diagram update * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu * diagram name fix * Fixes * Updates to index.mdx * Tech specs page corrections * Tech specs page rename * update link to tech specs * K8s - new pages + tech specs * k8s - manage peering connections * k8s L7 traffic management * Separated establish connection pages * Directory fixes * Usage clean up * k8s docs edits * Updated nav data * CodeBlock Component fix * filename * CodeBlockConfig removal * Redirects * Update k8s filenames * Reshuffle k8s tech specs for clarity, fmt yaml files * Update general cluster peering docs, reorder CLI > API > UI, cross link to kubernetes * Fix config rendering in k8s usage docs, cross link to general usage from k8s docs * fix legacy link * update k8s docs * fix nested list rendering * redirect fix * page error --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu Co-authored-by: Tu Nguyen * delete deprecated file --------- Co-authored-by: boruszak Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: David Yu From e388d6a09c2324b90b566e61f7a7b23c51276319 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 01:29:27 -0800 Subject: [PATCH 048/421] Backport of Refactor and move wal docs into release/1.15.x (#16432) * no-op commit due to failed cherry-picking * fix merge error --------- Co-authored-by: temp Co-authored-by: Tu Nguyen --- .../docs/agent/config/config-files.mdx | 92 +++++++++-- website/content/docs/agent/telemetry.mdx | 31 +++- .../docs/agent/wal-logstore/enable.mdx | 143 ++++++++++++++++++ .../content/docs/agent/wal-logstore/index.mdx | 48 ++++++ .../docs/agent/wal-logstore/monitoring.mdx | 85 +++++++++++ .../agent/wal-logstore/revert-to-boltdb.mdx | 76 ++++++++++ website/data/docs-nav-data.json | 21 +++ 7 files changed, 481 insertions(+), 15 deletions(-) create mode 100644 website/content/docs/agent/wal-logstore/enable.mdx create mode 100644 website/content/docs/agent/wal-logstore/index.mdx create mode 100644 website/content/docs/agent/wal-logstore/monitoring.mdx create mode 100644 website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 92047ce897e7..2992f562af00 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -1586,15 +1586,89 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." ## Raft Parameters -- `raft_boltdb` ((#raft_boltdb)) This is a nested object that allows configuring - options for Raft's BoltDB based log store. - - - `NoFreelistSync` ((#NoFreelistSync)) Setting this to `true` will disable - syncing the BoltDB freelist to disk within the raft.db file. Not syncing - the freelist to disk will reduce disk IO required for write operations - at the expense of potentially increasing start up time due to needing - to scan the db to discover where the free space resides within the file. - +- `raft_boltdb` ((#raft_boltdb)) **These fields are deprecated in Consul v1.15.0. + Use [`raft_logstore`](#raft_logstore) instead.** This is a nested + object that allows configuring options for Raft's BoltDB-based log store. + + - `NoFreelistSync` **This field is deprecated in Consul v1.15.0. Use the + [`raft_logstore.boltdb.no_freelist_sync`](#raft_logstore_boltdb_no_freelist_sync) field + instead.** Setting this to `true` disables syncing the BoltDB freelist + to disk within the raft.db file. Not syncing the freelist to disk + reduces disk IO required for write operations at the expense of potentially + increasing start up time due to needing to scan the db to discover where the + free space resides within the file. + +- `raft_logstore` ((#raft_logstore)) This is a nested object that allows + configuring options for Raft's LogStore component which is used to persist + logs and crucial Raft state on disk during writes. This was added in Consul + v1.15.0. + + - `backend` ((#raft_logstore_backend)) Specifies which storage + engine to use to persist logs. Valid options are `boltdb` or `wal`. Default + is `boltdb`. The `wal` option specifies an experimental backend that + should be used with caution. Refer to + [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `disable_log_cache` ((#raft_logstore_disable_log_cache)) Disables the in-memory cache for recent logs. We recommend using it for performance testing purposes, as no significant improvement has been measured when the cache is disabled. While the in-memory log cache theoretically prevents disk reads for recent logs, recent logs are also stored in the OS page cache, which does not slow either the `boltdb` or `wal` backend's ability to read them. + + - `verification` ((#raft_logstore_verification)) This is a nested object that + allows configuring the online verification of the LogStore. Verification + provides additional assurances that LogStore backends are correctly storing + data. It imposes low overhead on servers and is safe to run in + production. It is most useful when evaluating a new backend + implementation. + + Verification must be enabled on the leader to have any effect and can be + used with any backend. When enabled, the leader periodically writes a + special "checkpoint" log message that includes the checksums of all log entries + written to Raft since the last checkpoint. Followers that have verification + enabled run a background task for each checkpoint that reads all logs + directly from the LogStore and then recomputes the checksum. A report is output + as an INFO level log for each checkpoint. + + Checksum failure should never happen and indicate unrecoverable corruption + on that server. The only correct response is to stop the server, remove its + data directory, and restart so it can be caught back up with a correct + server again. Please report verification failures including details about + your hardware and workload via GitHub issues. Refer to + [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `enabled` ((#raft_logstore_verification_enabled)) - Set to `true` to + allow this Consul server to write and verify log verification checkpoints + when elected leader. + + - `interval` ((#raft_logstore_verification_interval)) - Specifies the time + interval between checkpoints. There is no default value. You must + configure the `interval` and set [`enabled`](#raft_logstore_verification_enabled) + to `true` to correctly enable intervals. We recommend using an interval + between `30s` and `5m`. The performance overhead is insignificant when the + interval is set to `5m` or less. + + - `boltdb` ((#raft_logstore_boltdb)) - Object that configures options for + Raft's `boltdb` backend. It has no effect if the `backend` is not `boltdb`. + + - `no_freelist_sync` ((#raft_logstore_boltdb_no_freelist_sync)) - Set to + `true` to disable storing BoltDB's freelist to disk within the + `raft.db` file. Disabling freelist syncs reduces the disk IO required + for write operations, but could potentially increase start up time + because Consul must scan the database to find free space + within the file. + + - - `wal` ((#raft_logstore_wal)) - Object that configures the `wal` backend. + Refer to [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `segment_size_mb` ((#raft_logstore_wal_segment_size_mb)) - Integer value + that represents the target size in MB for each segment file before + rolling to a new segment. The default value is `64` and is suitable for + most deployments. While a smaller value may use less disk space because you + can reclaim space by deleting old segments sooner, the smaller segment that results + may affect performance because safely rotating to a new file more + frequently can impact tail latencies. Larger values are unlikely + to improve performance significantly. We recommend using this + configuration for performance testing purposes. - `raft_protocol` ((#raft_protocol)) Equivalent to the [`-raft-protocol` command-line flag](/consul/docs/agent/config/cli-flags#_raft_protocol). diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index b234c01179a0..df8fdef15dc4 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -294,7 +294,7 @@ This metric should be monitored to ensure that the license doesn't expire to pre | Metric Name | Description | Unit | Type | | :-------------------------------- | :--------------------------------------------------------------- | :---- | :---- | -| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | +| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | | `consul.raft.boltdb.logsPerBatch` | Measures the number of logs being written per batch to the db. | logs | sample | | `consul.raft.boltdb.storeLogs` | Measures the amount of time spent writing logs to the db. | ms | timer | | `consul.raft.boltdb.writeCapacity` | Theoretical write capacity in terms of the number of logs that can be written per second. Each sample outputs what the capacity would be if future batched log write operations were similar to this one. This similarity encompasses 4 things: batch size, byte size, disk performance and boltdb performance. While none of these will be static and its highly likely individual samples of this metric will vary, aggregating this metric over a larger time window should provide a decent picture into how this BoltDB store can perform | logs/second | sample | @@ -337,11 +337,12 @@ indicator of an actual issue, this metric can be used to diagnose why the `consu is high. If Bolt DB log storage performance becomes an issue and is caused by free list management then setting -[`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) to `true` in the server's configuration +[`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) to `true` in the server's configuration may help to reduce disk IO and log storage operation times. Disabling free list syncing will however increase the startup time for a server as it must scan the raft.db file for free space instead of loading the already populated free list structure. +Consul includes an experiment backend configuration that you can use instead of BoldDB. Refer to [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) for more information. ## Metrics Reference @@ -418,7 +419,7 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.applied_index` | Represents the raft applied index. | index | gauge | | `consul.raft.apply` | Counts the number of Raft transactions occurring over the interval, which is a general indicator of the write load on the Consul servers. | raft transactions / interval | counter | | `consul.raft.barrier` | Counts the number of times the agent has started the barrier i.e the number of times it has issued a blocking call, to ensure that the agent has all the pending operations that were queued, to be applied to the agent's FSM. | blocks / interval | counter | -| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | +| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | | `consul.raft.boltdb.freePageBytes` | Represents the number of bytes of free space within the raft.db file. | bytes | gauge | | `consul.raft.boltdb.getLog` | Measures the amount of time spent reading logs from the db. | ms | timer | | `consul.raft.boltdb.logBatchSize` | Measures the total size in bytes of logs being written to the db in a single batch. | bytes | sample | @@ -452,6 +453,13 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.last_index` | Represents the raft applied index. | index | gauge | | `consul.raft.leader.dispatchLog` | Measures the time it takes for the leader to write log entries to disk. | ms | timer | | `consul.raft.leader.dispatchNumLogs` | Measures the number of logs committed to disk in a batch. | logs | gauge | +| `consul.raft.logstore.verifier.checkpoints_written` | Counts the number of checkpoint entries written to the LogStore. | checkpoints | counter | +| `consul.raft.logstore.verifier.dropped_reports` | Counts how many times the verifier routine was still busy when the next checksum came in and so verification for a range was skipped. If you see this happen, consider increasing the interval between checkpoints with [`raft_logstore.verification.interval`](/consul/docs/agent/config/config-files#raft_logstore_verification) | reports dropped | counter | +| `consul.raft.logstore.verifier.ranges_verified` | Counts the number of log ranges for which a verification report has been completed. Refer to [Monitor Raft metrics and logs for WAL +](/consul/docs/agent/wal-logstore/monitoring) for more information. | log ranges verifications | counter | +| `consul.raft.logstore.verifier.read_checksum_failures` | Counts the number of times a range of logs between two check points contained at least one disk corruption. Refer to [Monitor Raft metrics and logs for WAL +](/consul/docs/agent/wal-logstore/monitoring) for more information. | disk corruptions | counter | +| `consul.raft.logstore.verifier.write_checksum_failures` | Counts the number of times a follower has a different checksum to the leader at the point where it writes to the log. This could be caused by either a disk-corruption on the leader (unlikely) or some other corruption of the log entries in-flight. | in-flight corruptions | counter | | `consul.raft.leader.lastContact` | Measures the time since the leader was last able to contact the follower nodes when checking its leader lease. It can be used as a measure for how stable the Raft timing is and how close the leader is to timing out its lease.The lease timeout is 500 ms times the [`raft_multiplier` configuration](/consul/docs/agent/config/config-files#raft_multiplier), so this telemetry value should not be getting close to that configured value, otherwise the Raft timing is marginal and might need to be tuned, or more powerful servers might be needed. See the [Server Performance](/consul/docs/install/performance) guide for more details. | ms | timer | | `consul.raft.leader.oldestLogAge` | The number of milliseconds since the _oldest_ log in the leader's log store was written. This can be important for replication health where write rate is high and the snapshot is large as followers may be unable to recover from a restart if restoring takes longer than the minimum value for the current leader. Compare this with `consul.raft.fsm.lastRestoreDuration` and `consul.raft.rpc.installSnapshot` to monitor. In normal usage this gauge value will grow linearly over time until a snapshot completes on the leader and the log is truncated. Note: this metric won't be emitted until the leader writes a snapshot. After an upgrade to Consul 1.10.0 it won't be emitted until the oldest log was written after the upgrade. | ms | gauge | | `consul.raft.replication.heartbeat` | Measures the time taken to invoke appendEntries on a peer, so that it doesn't timeout on a periodic basis. | ms | timer | @@ -476,9 +484,20 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.state.follower` | Counts the number of times an agent has entered the follower mode. This happens when a new agent joins the cluster or after the end of a leader election. | follower state entered / interval | counter | | `consul.raft.transition.heartbeat_timeout` | The number of times an agent has transitioned to the Candidate state, after receive no heartbeat messages from the last known leader. | timeouts / interval | counter | | `consul.raft.verify_leader` | This metric doesn't have a direct correlation to the leader change. It just counts the number of times an agent checks if it is still the leader or not. For example, during every consistent read, the check is done. Depending on the load in the system, this metric count can be high as it is incremented each time a consistent read is completed. | checks / interval | Counter | -| `consul.rpc.accept_conn` | Increments when a server accepts an RPC connection. | connections | counter | -| `consul.rpc.rate_limit.exceeded` | Number of rate limited requests. Only increments when `rate_limits.mode` is set to `permissive` or `enforcing`. | requests | counter | -| `consul.rpc.rate_limit.log_dropped` | Number of rate limited requests logs dropped for performance reasons. Only increments when `rate_limits.mode` is set to `permissive` or `enforcing` and the log is unable to print the number of excessive requests. | log lines | counter | +| `consul.raft.wal.head_truncations` | Counts how many log entries have been truncated from the head - i.e. the oldest entries. by graphing the rate of change over time you can see individual truncate calls as spikes. | logs entries truncated | counter | +| `consul.raft.wal.last_segment_age_seconds` | A gauge that is set each time we rotate a segment and describes the number of seconds between when that segment file was first created and when it was sealed. this gives a rough estimate how quickly writes are filling the disk. | seconds | gauge | +| `consul.raft.wal.log_appends` | Counts the number of calls to StoreLog(s) i.e. number of batches of entries appended. | calls | counter | +| `consul.raft.wal.log_entries_read` | Counts the number of log entries read. | log entries read | counter | +| `consul.raft.wal.log_entries_written` | Counts the number of log entries written. | log entries written | counter | +| `consul.raft.wal.log_entry_bytes_read` | Counts the bytes of log entry read from segments before decoding. actual bytes read from disk might be higher as it includes headers and index entries and possible secondary reads for large entries that don't fit in buffers. | bytes | counter | +| `consul.raft.wal.log_entry_bytes_written` | Counts the bytes of log entry after encoding with Codec. Actual bytes written to disk might be slightly higher as it includes headers and index entries. | bytes | counter | +| `consul.raft.wal.segment_rotations` | Counts how many times we move to a new segment file. | rotations | counter | +| `consul.raft.wal.stable_gets` | Counts how many calls to StableStore.Get or GetUint64. | calls | counter | +| `consul.raft.wal.stable_sets` | Counts how many calls to StableStore.Set or SetUint64. | calls | counter | +| `consul.raft.wal.tail_truncations` | Counts how many log entries have been truncated from the head - i.e. the newest entries. by graphing the rate of change over time you can see individual truncate calls as spikes. | logs entries truncated | counter | +| `consul.rpc.accept_conn` | Increments when a server accepts an RPC connection. | connections | counter | +| `consul.rpc.rate_limit.exceeded` | Increments whenever an RPC is over a configured rate limit. In permissive mode, the RPC is still allowed to proceed. | RPCs | counter | +| `consul.rpc.rate_limit.log_dropped` | Increments whenever a log that is emitted because an RPC exceeded a rate limit gets dropped because the output buffer is full. | log messages dropped | counter | | `consul.catalog.register` | Measures the time it takes to complete a catalog register operation. | ms | timer | | `consul.catalog.deregister` | Measures the time it takes to complete a catalog deregister operation. | ms | timer | | `consul.server.isLeader` | Track if a server is a leader(1) or not(0) | 1 or 0 | gauge | diff --git a/website/content/docs/agent/wal-logstore/enable.mdx b/website/content/docs/agent/wal-logstore/enable.mdx new file mode 100644 index 000000000000..a4a89f70c86e --- /dev/null +++ b/website/content/docs/agent/wal-logstore/enable.mdx @@ -0,0 +1,143 @@ +--- +layout: docs +page_title: Enable the experimental WAL LogStore backend +description: >- + Learn how to safely configure and test the experimental WAL backend in your Consul deployment. +--- + +# Enable the experimental WAL LogStore backend + +This topic describes how to safely configure and test the WAL backend in your Consul deployment. + +The overall process for enabling the WAL LogStore backend for one server consists of the following steps. In production environments, we recommend starting by enabling the backend on a single server . If you eventually choose to expand the test to further servers, you must repeat these steps for each one. + +1. Enable log verification. +1. Select target server to enable WAL. +1. Stop target server gracefully. +1. Remove data directory from target server. +1. Update target server's configuration. +1. Start the target server. +1. Monitor target server raft metrics and logs. + +!> **Experimental feature:** The WAL LogStore backend is experimental. + +## Requirements + +- Consul v1.15 or later is required for all servers in the datacenter. Refer to the [standard upgrade procedure](/consul/docs/upgrading/general-process) and the [1.15 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-15-x) for additional information. +- A Consul cluster with at least three nodes are required to safely test the WAL backend without downtime. + +We recommend taking the following additional measures: + +- Take a snapshot prior to testing. +- Monitor Consul server metrics and logs, and set an alert on specific log events that occur when WAL is enabled. Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for more information. +- Enable WAL in a pre-production environment and run it for a several days before enabling it in production. + +## Risks + +While their likelihood remains low to very low, be aware of the following risks before implementing the WAL backend: + + - If WAL corrupts data on a Consul server agent, the server data cannot be recovered. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - WAL may corrupt data or contain a defect that causes the server to panic and crash. WAL may not restart if the defect recurs when WAL reads from the logs on startup. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - If WAL corrupts data, clients may read corrupted data from the Consul server, such as invalid IP addresses or unmatched tokens. This outcome is unlikely even if a recurring defect causes WAL to corrupt data because replication uses objects cached in memory instead of reads from disk. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - If you enable a Consul OSS server to use WAL or enable WAL on a voting server with Consul Enterprise, WAL may corrupt the server's state, become the leader, and replicate the corrupted state to all other servers. In this case, restoring from backup is required to recover a completely uncorrupted state. Test WAL on a non-voting server in Enterprise to prevent this outcome. You can add a new non-voting server to the cluster to test with if there are no existing ones. + +## Enable log verification + +You must enable log verification on all voting servers in Enterprise and all servers in OSS because the leader writes verification checkpoints. + +1. On each voting server, add the following to the server's configuration file: + + ```hcl + raft_logstore { + verification { + enabled = true + interval = "60s" + } + } + ``` + +1. Restart the server to apply the changes. The `consul reload` command is not sufficient to apply `raft_logstore` configuration changes. +1. Run the `consul operator raft list-peers` command to wait for each server to become a healthy voter before moving on to the next. This may take a few minutes for large snapshots. + +When complete, the server's logs should contain verifier reports that appear like the following example: + +```log hideClipboard +2023-01-31T14:44:31.174Z [INFO] agent.server.raft.logstore.verifier: verification checksum OK: elapsed=488.463268ms leaderChecksum=f15db83976f2328c rangeEnd=357802 rangeStart=298132 readChecksum=f15db83976f2328c +``` + +## Select target server to enable WAL + +If you are using Consul OSS or Consul Enterprise without non-voting servers, select a follower server to enable WAL. As noted in [Risks](#risks), Consul Enterprise users with non-voting servers should first select a non-voting server, or consider adding another server as a non-voter to test on. + +Retrieve the current state of the servers by running the following command: + +```shell-session +$ consul operator raft list-peers +``` + +## Stop target server + +Stop the target server gracefully. For example, if you are using `systemd`, +run the following command: + +```shell-session +$ systemctl stop consul +``` + +If your environment uses configuration management automation that might interfere with this process, such as Chef or Puppet, you must disable them until you have completely enabled WAL as a storage backend. + +## Remove data directory from target server + +Temporarily moving the data directory to a different location is less destructive than deleting it. We recommend moving it in cases where you unsuccessfully enable WAL. Do not use the old data directory (`/data-dir/raft.bak`) for recovery after restarting the server. We recommend eventually deleting the old directory. + +The following example assumes the `data_dir` in the server's configuration is `/data-dir` and renames it to `/data-dir.bak`. + +```shell-session +$ mv /data-dir/raft /data-dir/raft.bak +``` + +When switching backends, you must always remove _the entire raft directory_, not just the `raft.db` file or `wal` directory. The log must always be consistent with the snapshots to avoid undefined behavior or data loss. + +## Update target server configuration + +Add the following to the target server's configuration file: + +```hcl +raft_logstore { + backend = "wal" + verification { + enabled = true + interval = "60s" + } +} +``` + +## Start target server + +Start the target server. For example, if you are using `systemd`, run the following command: + +```shell-session +$ systemctl start consul +``` + +Watch for the server to become a healthy voter again. + +```shell-session +$ consul operator raft list-peers +``` + +## Monitor target server Raft metrics and logs + +Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for details. + +We recommend leaving the cluster in the test configuration for several days or weeks, as long as you observe no errors. An extended test provides more confidence that WAL operates correctly under varied workloads and during routine server restarts. If you observe any errors, end the test immediately and report them. + +If you disabled configuration management automation, consider reenabling it during the testing phase to pick up other updates for the host. You must ensure that it does not revert the Consul configuration file and remove the altered backend configuration. One way to do this is add the `raft_logstore` block to a separate file that is not managed by your automation. This file can either be added to the directory if [`-config-dir`](/consul/docs/agent/config/cli-flags#_config_dir) is used or as an additional [`-config-file`](/consul/docs/agent/config/cli-flags#_config_file) argument. + +## Next steps + +- If you observe any verification errors, performance anomalies, or other suspicious behavior from the target server during the test, you should immediately follow [the procedure to revert back to BoltDB](/consul/docs/agent/wal-logstore/revert-to-boltdb). Report failures through GitHub. + +- If you do not see errors and would like to expand the test further, you can repeat the above procedure on another target server. We suggest waiting after each test expansion and slowly rolling WAL out to other parts of your environment. Once the majority of your servers use WAL, any bugs not yet discovered may result in cluster unavailability. + +- If you wish to permanently enable WAL on all servers, repeat the steps described in this topic for each server. Even if `backend = "wal"` is set in logs, servers continue to use BoltDB if they find an existing raft.db file in the data directory. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/index.mdx b/website/content/docs/agent/wal-logstore/index.mdx new file mode 100644 index 000000000000..914d6602a9e9 --- /dev/null +++ b/website/content/docs/agent/wal-logstore/index.mdx @@ -0,0 +1,48 @@ +--- +layout: docs +page_title: WAL LogStore Backend Overview +description: >- + The experimental WAL (write-ahead log) LogStore backend shipped in Consul 1.15 is intended to replace the BoltDB backend, improving performance and log storage issues. +--- + +# Experimental WAL LogStore backend overview + +This topic provides an overview of the experimental WAL (write-ahead log) LogStore backend. + +!> **Experimental feature:** The WAL LogStore backend is experimental. + +## WAL versus BoltDB + +WAL implements a traditional log with rotating, append-only log files. WAL resolves many issues with the existing `LogStore` provided by the BoltDB backend. The BoltDB `LogStore` is a copy-on-write BTree, which is not optimized for append-only, write-heavy workloads. + +### BoltDB storage scalability issues + +The existing BoltDB log store inefficiently stores append-only logs to disk because it was designed as a full key-value database. It is a single file that only ever grows. Deleting the oldest logs, which Consul does regularly when it makes new snapshots of the state, leaves free space in the file. The free space must be tracked in a `freelist` so that BoltDB can reuse it on future writes. By contrast, a simple segmented log can delete the oldest log files from disk. + +A burst of writes at double or triple the normal volume can suddenly cause the log file to grow to several times its steady-state size. After Consul takes the next snapshot and truncates the oldest logs, the resulting file is mostly empty space. + +To track the free space, Consul must write extra metadata to disk with every write. The metadata is proportional to the amount of free pages, so after a large burst write latencies tend to increase. In some cases, the latencies cause serious performance degradation to the cluster. + +To mitigate risks associated with sudden bursts of log data, Consul tries to limit lots of logs from accumulating in the LogStore. Significantly larger BoltDB files are slower to append to because the tree is deeper and freelist larger. For this reason, Consul's default options associated with snapshots, truncating logs, and keeping the log history have been aggressively set toward keeping BoltDB small rather than using disk IO optimally. + +But the larger the file, the more likely it is to have a large freelist or suddenly form one after a burst of writes. For this reason, the many of Consul's default options asssociated with snapshots, truncating logs, and keeping the log history aggressively keep BoltDT small rather than uisng disk IO more efficiently. + +Other reliability issues, such as [raft replication capacity issues](/consul/docs/agent/telemetry#raft-replication-capacity-issues), are much simpler to solve without the performance concerns caused by storing more logs in BoltDB. + +### WAL approaches storage issues differently + +When directly measured, WAL is more performant than BoltDB because it solves a simpler storage problem. Despite this, some users may not notice a significant performance improvement from the upgrade with the same configuration and workload. In this case, the benefit of WAL is that retaining more logs does not affect write performance. As a result, strategies for reducing disk IO with slower snapshots or for keeping logs to permit slower followers to catch up with cluster state are all possible, increasing the reliability of the deployment. + +## WAL quality assurance + +The WAL backend has been tested thoroughly during development: + +- Every component in the WAL, such as [metadata management](https://github.com/hashicorp/raft-wal/blob/main/types/meta.go), [log file encoding](https://github.com/hashicorp/raft-wal/blob/main/types/segment.go) to actual [file-system interaction](https://github.com/hashicorp/raft-wal/blob/main/types/vfs.go) are abstracted so unit tests can simulate difficult-to-reproduce disk failures. + +- We used the [application-level intelligent crash explorer (ALICE)](https://github.com/hashicorp/raft-wal/blob/main/alice/README.md) to exhaustively simulate thousands of possible crash failure scenarios. WAL correctly recovered from all scenarios. + +- We ran hundreds of tests in a performance testing cluster with checksum verification enabled and did not detect data loss or corruption. We will continue testing before making WAL the default backend. + +We are aware of how complex and critical disk-persistence is for your data. + +We hope that many users at different scales will try WAL in their environments after upgrading to 1.15 or later and report success or failure so that we can confidently replace BoltDB as the default for new clusters in a future release. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/monitoring.mdx b/website/content/docs/agent/wal-logstore/monitoring.mdx new file mode 100644 index 000000000000..5be765cf408b --- /dev/null +++ b/website/content/docs/agent/wal-logstore/monitoring.mdx @@ -0,0 +1,85 @@ +--- +layout: docs +page_title: Monitor Raft metrics and logs for WAL +description: >- + Learn how to monitor Raft metrics emitted the experimental WAL (write-ahead log) LogStore backend shipped in Consul 1.15. +--- + +# Monitor Raft metrics and logs for WAL + +This topic describes how to monitor Raft metrics and logs if you are testing the WAL backend. We strongly recommend monitoring the Consul cluster, especially the target server, for evidence that the WAL backend is not functioning correctly. Refer to [Enable the experimental WAL LogStore backend](/consul/docs/agent/wal-logstore/index) for additional information about the WAL backend. + +!> **Upgrade warning:** The WAL LogStore backend is experimental. + +## Monitor for checksum failures + +Log store verification failures on any server, regardless of whether you are running the BoltDB or WAL backed, are unrecoverable errors. Consul may report the following errors in logs. + +### Read failures: Disk Corruption + +```log hideClipboard +2022-11-15T22:41:23.546Z [ERROR] agent.raft.logstore: verification checksum FAILED: storage corruption rangeStart=1234 rangeEnd=3456 leaderChecksum=0xc1... readChecksum=0x45... +``` + +This indicates that the server read back data that is different from what it wrote to disk. This indicates corruption in the storage backend or filesystem. + +For convenience, Consul also increments a metric `consul.raft.logstore.verifier.read_checksum_failures` when this occurs. + +### Write failures: In-flight Corruption + +The following error indicates that the checksum on the follower did not match the leader when the follower received the logs. The error implies that the corruption happened in the network or software and not the log store: + +```log hideClipboard +2022-11-15T22:41:23.546Z [ERROR] agent.raft.logstore: verification checksum FAILED: in-flight corruption rangeStart=1234 rangeEnd=3456 leaderChecksum=0xc1... followerWriteChecksum=0x45... +``` + +It is unlikely that this error indicates an issue with the storage backend, but you should take the same steps to resolve and report it. + +The `consul.raft.logstore.verifier.write_checksum_failures` metric increments when this error occurs. + +## Resolve checksum failures + +If either type of corruption is detected, complete the instructions for [reverting to BoltDB](/consul/docs/agent/wal-logstore/revert-to-boltdb). If the server already uses BoltDB, the errors likely indicate a latent bug in BoltDB or a bug in the verification code. In both cases, you should follow the revert instructions. + +Report all verification failures as a [GitHub +issue](https://github.com/hashicorp/consul/issues/new?assignees=&labels=&template=bug_report.md&title=WAL:%20Checksum%20Failure). + +In your report, include the following: + - Details of your server cluster configuration and hardware + - Logs around the failure message + - Context for how long they have been running the configuration + - Any metrics or description of the workload you have. For example, how many raft + commits per second. Also include the performance metrics described on this page. + +We recommend setting up an alert on Consul server logs containing `verification checksum FAILED` or on the `consul.raft.logstore.verifier.{read|write}_checksum_failures` metrics. The sooner you respond to a corrupt server, the lower the chance of any of the [potential risks](/consul/docs/agent/wal-logstore/enable#risks) causing problems in your cluster. + +## Performance metrics + +The key performance metrics to watch are: + +- `consul.raft.commitTime` measures the time to commit new writes on a quorum of + servers. It should be the same or lower after deploying WAL. Even if WAL is + faster for your workload and hardware, it may not be reflected in `commitTime` + until enough followers are using WAL that the leader does not have to wait for + two slower followers in a cluster of five to catch up. + +- `consul.raft.rpc.appendEntries.storeLogs` measures the time spent persisting + logs to disk on each _follower_. It should be the same or lower for + WAL-enabled followers. + +- `consul.raft.replication.appendEntries.rpc` measures the time taken for each + `AppendEntries` RPC from the leader's perspective. If this is significantly + higher than `consul.raft.rpc.appendEntries` on the follower, it indicates a + known queuing issue in the Raft library and is unrelated to the backend. + Followers with WAL enabled should not be slower than the others. You can + determine which follower is associated with which metric by running the + `consul operator raft list-peers` command and matching the + `peer_id` label value to the server IDs listed. + +- `consul.raft.compactLogs` measures the time take to truncate the logs after a + snapshot. WAL-enabled servers should not be slower than BoltDB servers. + +- `consul.raft.leader.dispatchLog` measures the time spent persisting logs to + disk on the _leader_. It is only relevant if a WAL-enabled server becomes a + leader. It should be the same or lower than before when the leader was using + BoltDB. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx b/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx new file mode 100644 index 000000000000..2bd0638b7cd3 --- /dev/null +++ b/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx @@ -0,0 +1,76 @@ +--- +layout: docs +page_title: Revert to BoltDB +description: >- + Learn how to revert Consul to the BoltDB backend after enabled the WAL (write-ahead log) LogStore backend shipped in Consul 1.15. +--- + +# Revert storage backend to BoltDB from WAL + +This topic describes how to revert your Consul storage backend from the experimental WAL LogStore backend to the default BoltDB. + +The overall process for reverting to BoltDB consists of the following steps. Repeat the steps for all Consul servers that you need to revert. + +1. Stop target server gracefully. +1. Remove data directory from target server. +1. Update target server's configuration. +1. Start target server. + +## Stop target server gracefully + +Stop the target server gracefully. For example, if you are using `systemd`, +run the following command: + +```shell-session +$ systemctl stop consul +``` + +If your environment uses configuration management automation that might interfere with this process, such as Chef or Puppet, you must disable them until you have completely revereted the storage backend. + +## Remove data directory from target server + +Temporarily moving the data directory to a different location is less destructive than deleting it. We recommend moving the data directory instead of deleted it in cases where you unsuccessfully enable WAL. Do not use the old data directory (`/data-dir/raft.bak`) for recovery after restarting the server. We recommend eventually deleting the old directory. + +The following example assumes the `data_dir` in the server's configuration is `/data-dir` and renames it to `/data-dir.wal.bak`. + +```shell-session +$ mv /data-dir/raft /data-dir/raft.wal.bak +``` + +When switching backend, you must always remove _the entire raft directory_ not just the `raft.db` file or `wal` directory. This is because the log must always be consistent with the snapshots to avoid undefined behavior or data loss. + +## Update target server's configuration + +Modify the `backend` in the target server's configuration file: + +```hcl +raft_logstore { + backend = "boltdb" + verification { + enabled = true + interval = "60s" + } +} +``` + +## Start target server + +Start the target server. For example, if you are using `systemd`, run the following command: + +```shell-session +$ systemctl start consul +``` + +Watch for the server to become a healthy voter again. + +```shell-session +$ consul operator raft list-peers +``` + +### Clean up old data directories + +If necessary, clean up any `raft.wal.bak` directories. Replace `/data-dir` with the value you specified in your configuration file. + +```shell-session +$ rm /data-dir/raft.bak +``` \ No newline at end of file diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index f229450164a0..44dff90e2f1e 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -851,6 +851,27 @@ "title": "RPC", "path": "agent/rpc", "hidden": true + }, + { + "title": "Experimental WAL LogStore", + "routes": [ + { + "title": "Overview", + "path": "agent/wal-logstore" + }, + { + "title": "Enable WAL LogStore backend", + "path": "agent/wal-logstore/enable" + }, + { + "title": "Monitor Raft metrics and logs for WAL", + "path": "agent/wal-logstore/monitoring" + }, + { + "title": "Revert to BoltDB", + "path": "agent/wal-logstore/revert-to-boltdb" + } + ] } ] }, From c7cbb3e884cd5b4b2577f7896eceba6876582ab5 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 09:28:26 -0800 Subject: [PATCH 049/421] backport of commit 6f46f6396dd7250902cb8146df044fece1fbdf6e (#16438) Co-authored-by: Tu Nguyen --- .../docs/agent/limits/set-global-traffic-rate-limits.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx index 6e14ea2b1cba..fc655833865c 100644 --- a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx +++ b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx @@ -12,7 +12,7 @@ Rate limits apply to each Consul server separately and are intended to limit the Because all requests coming to a Consul server eventually perform an RPC or an internal gRPC request, these limits also apply to other user interfaces, such as the HTTP API interface, the CLI, and the external gRPC endpoint for services in the Consul service mesh. -Refer to [Initialize Rate Limit Settings]() for additional information about right-sizing your gRPC request configurations. +Refer to [Initialize Rate Limit Settings](/consul/docs/agent/limits/init-rate-limits) for additional information about right-sizing your gRPC request configurations. ## Set a global rate limit for a Consul server Configure the following settings in your Consul server configuration to limit the RPC and gRPC traffic rates. From 8c99be8dca31f03a514509fa31fa21623cee7d9e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 09:58:31 -0800 Subject: [PATCH 050/421] backport of commit 1a0bc58d7224ca687b98d34a00a5c9c19d93dea9 (#16439) Co-authored-by: Bryce Kalow --- .../content/docs/connect/config-entries/service-splitter.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/connect/config-entries/service-splitter.mdx b/website/content/docs/connect/config-entries/service-splitter.mdx index 34ea9597e218..5d94a4e38a22 100644 --- a/website/content/docs/connect/config-entries/service-splitter.mdx +++ b/website/content/docs/connect/config-entries/service-splitter.mdx @@ -1,4 +1,4 @@ ---- +--- layout: docs page_title: Service Splitter Configuration Entry Reference description: >- @@ -784,4 +784,4 @@ spec: - \ No newline at end of file + From 87394099dc9de749689aa459c29572f05e37bb42 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 10:22:59 -0800 Subject: [PATCH 051/421] Backport of Add envoy extension docs into release/1.15.x (#16394) * backport of commit 98096280c093133747784683c4985598987cc688 * fix merge conflicts --------- Co-authored-by: Tu Nguyen Co-authored-by: Tu Nguyen --- .../connect/config-entries/proxy-defaults.mdx | 3 +- .../proxies/envoy-extensions/index.mdx | 31 +++ .../proxies/envoy-extensions/usage/lambda.mdx | 161 +++++++++++++++ .../proxies/envoy-extensions/usage/lua.mdx | 185 ++++++++++++++++++ website/data/docs-nav-data.json | 22 +++ 5 files changed, 401 insertions(+), 1 deletion(-) create mode 100644 website/content/docs/connect/proxies/envoy-extensions/index.mdx create mode 100644 website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx create mode 100644 website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx diff --git a/website/content/docs/connect/config-entries/proxy-defaults.mdx b/website/content/docs/connect/config-entries/proxy-defaults.mdx index 5ac09c102f09..cb9b9991712f 100644 --- a/website/content/docs/connect/config-entries/proxy-defaults.mdx +++ b/website/content/docs/connect/config-entries/proxy-defaults.mdx @@ -351,7 +351,8 @@ spec: { name: 'EnvoyExtensions', type: 'array: []', - description: `A list of extensions to modify Envoy proxy configuration.`, + description: `A list of extensions to modify Envoy proxy configuration.

+ Applying \`EnvoyExtensions\` to \`ProxyDefaults\` may produce unintended consequences. We recommend enabling \`EnvoyExtensions\` with [\`ServiceDefaults\`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) in most cases.`, children: [ { name: 'Name', diff --git a/website/content/docs/connect/proxies/envoy-extensions/index.mdx b/website/content/docs/connect/proxies/envoy-extensions/index.mdx new file mode 100644 index 000000000000..844ab0706c3a --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/index.mdx @@ -0,0 +1,31 @@ +--- +layout: docs +page_title: Envoy Extensions +description: >- + Learn how Envoy extensions enables you to add support for additional Envoy features without modifying the Consul codebase. +--- + +# Envoy extensions overview + +This topic provides an overview of Envoy extensions in Consul service mesh deployments. You can modify Consul-generated Envoy resources to add additional functionality without modifying the Consul codebase. + +## Introduction + +Consul supports two methods for modifying Envoy behavior. You can either modify the Envoy resources Consul generates through [escape hatches](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) or configure your services to use Envoy extensions using the `EnvoyExtension` parameter. Implementing escape hatches requires rewriting the Envoy resources so that they are compatible with Consul, a task that also requires understanding how Consul names Envoy resources and enforces intentions. + +Instead of modifying Consul code, you can configure your services to use Envoy extensions through the `EnvoyExtensions` field. This field is definable in [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) and [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) configuration entries. + +## Supported extensions + +Envoy extensions enable additional service mesh functionality in Consul by changing how the sidecar proxies behave. Extensions invoke custom code when traffic passes through an Envoy proxy. Consul supports the following extensions: + +- Lua +- Lambda + +### Lua + +The `lua` Envoy extension enables HTTP Lua filters in your Consul Envoy proxies. It allows you to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. Refer to the [`lua`](/consul/docs/proxies/envoy-extensions/usage/lua) documentation for more information. + +### Lambda + +The `lambda` Envoy extension enables services to make requests to AWS Lambda functions through the mesh as if they are a normal part of the Consul catalog. Refer to the [`lambda`](/consul/docs/proxies/envoy-extensions/usage/lambda) documentation for more information. diff --git a/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx b/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx new file mode 100644 index 000000000000..ce1554641969 --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx @@ -0,0 +1,161 @@ +--- +layout: docs +page_title: Lambda Envoy Extension +description: >- + Learn how the `lambda` Envoy extension enables Consul to join AWS Lambda functions to its service mesh. +--- + +# Invoke Lambda functions in Envoy proxy + +The Lambda Envoy extension configures outbound traffic on upstream dependencies allowing mesh services to properly invoke AWS Lambda functions. Lambda functions appear in the catalog as any other Consul service. + +You can only enable the Lambda extension through `service-defaults`. This is because the Consul uses the `service-defaults` configuration entry name as the catalog name for the Lambda functions. + +## Specification + +The Lambda Envoy extension has the following arguments: + +| Arguments | Description | +| -------------------- | ------------------------------------------------------------------------------------------------ | +| `ARN` | Specifies the [AWS ARN](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) for the service's Lambda. | +| `InvocationMode` | Determines if Consul configures the Lambda to be invoked using the synchronous or asynchronous [invocation mode](https://docs.aws.amazon.com/lambda/latest/operatorguide/invocation-modes.html). | +| `PayloadPassthrough` | Determines if the body Envoy receives is converted to JSON or directly passed to Lambda. | + +Be aware that unlike [manual lambda registration](/consul/docs/lambda/registration/manual#supported-meta-fields), region is inferred from the ARN when specified through an Envoy extension. + +## Workflow + +There are two steps to configure the Lambda Envoy extension: + +1. Configure EnvoyExtensions through `service-defaults`. +1. Apply the configuration entry. + +### Configure `EnvoyExtensions` + +To use the Lambda Envoy extension, you must configure and apply a `service-defaults` configuration entry. Consul uses the name of the entry as the Consul service name for the Lambdas in the catalog. Downstream services also use the name to invoke the Lambda. + +The following example configures the Lambda Envoy extension to create a service named `lambda` in the mesh that can invoke the associated Lambda function. + + + + + +```hcl +Kind = "service-defaults" +Name = "lambdaInvokingApp" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/aws/lambda" + Arguments = { + ARN = "arn:aws:lambda:us-west-2:111111111111:function:lambda-1234" + } +} +``` + + + + + + +```hcl +{ + "kind": "service-defaults", + "name": "lambdaInvokingApp", + "protocol": "http", + "envoy_extensions": [{ + "name": "builtin/aws/lambda", + "arguments": { + "arn": "arn:aws:lambda:us-west-2:111111111111:function:lambda-1234" + } + }] +} + +``` + + + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: lambda +spec: + protocol: http + envoyExtensions: + name = "builtin/aws/lambda" + arguments: + arn: arn:aws:lambda:us-west-2:111111111111:function:lambda-1234 +``` + + + + + + +For a full list of parameters for `EnvoyExtensions`, refer to the [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) and [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries reference documentation. + +~> **Note:** You can only enable the Lambda extension through `service-defaults`. + +Refer to [Configuration specification](#configuration-specification) section to find a full list of arguments for the Lambda Envoy extension. + +### Apply the configuration entry + +Apply the `service-defaults` configuration entry. + + + + +```shell-session +$ consul config write lambda-envoy-extension.hcl +``` + + + + +```shell-session +$ consul config write lambda-envoy-extension.json +``` + + + + +```shell-session +$ kubectl apply lambda-envoy-extension.yaml +``` + + + + +## Examples + +In the following example, the Lambda Envoy extension adds a single Lambda function running in two regions into the mesh. Then, you can use the `lambda` service name to invoke it, as if it was any other service in the mesh. + + + +```hcl +Kind = "service-defaults" +Name = "lambda" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/aws/lambda" + + Arguments = { + payloadPassthrough: false + arn: arn:aws:lambda:us-west-2:111111111111:function:lambda-1234 + } +} +EnvoyExtensions { + Name = "builtin/aws/lambda" + + Arguments = { + payloadPassthrough: false + arn: arn:aws:lambda:us-east-1:111111111111:function:lambda-1234 + } +} +``` + + \ No newline at end of file diff --git a/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx new file mode 100644 index 000000000000..a68e7a822518 --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx @@ -0,0 +1,185 @@ +--- +layout: docs +page_title: Lua Envoy Extension +description: >- + Learn how the `lua` Envoy extension enables Consul to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. +--- + +# Run Lua scripts in Envoy proxy + +The Lua Envoy extension enables the [HTTP Lua filter](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/lua_filter) in your Consul Envoy proxies, letting you run Lua scripts when requests and responses pass through Consul-generated Envoy resources. + +Envoy filters support setting and getting dynamic metadata, allowing a filter to share state information with subsequent filters. To set dynamic metadata, configure the HTTP Lua filter. Users can call `streamInfo:dynamicMetadata()` from Lua scripts to get the request's dynamic metadata. + +## Configuration specifications + +To use the Lua Envoy extension, configure the following arguments in the `EnvoyExtensions` block: + +| Arguments | Description | +| -------------- | ------------------------------------------------------------------------------------------------ | +| `ProxyType` | Determines the proxy type the extension applies to. The only supported value is `connect-proxy`. | +| `ListenerType` | Specifies if the extension is applied to the `inbound` or `outbound` listener. | +| `Script` | The Lua script that is configured to run by the HTTP Lua filter. | + +## Workflow + +There are two steps to configure the Lua Envoy extension: + +1. Configure EnvoyExtensions through `service-defaults` or `proxy-defaults`. +1. Apply the configuration entry. + +### Configure `EnvoyExtensions` + +To use Envoy extensions, you must configure and apply a `proxy-defaults` or `service-defaults` configuration entry with the Envoy extension. + +- When you configure Envoy extensions on `proxy-defaults`, they apply to every service. +- When you configure Envoy extensions on `service-defaults`, they apply to a specific service. + +Consul applies Envoy extensions configured in `proxy-defaults` before it applies extensions in `service-defaults`. As a result, the Envoy extension configuration in `service-defaults` may override configurations in `proxy-defaults`. + +The following example configures the Lua Envoy extension on every service by using the `proxy-defaults`. + + + + + +```hcl +Kind = "proxy-defaults" +Name = "global" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/lua" + + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOS +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-service", m["service"]) + request_handle:headers():add("x-consul-namespace", m["namespace"]) + request_handle:headers():add("x-consul-datacenter", m["datacenter"]) + request_handle:headers():add("x-consul-trust-domain", m["trust-domain"]) +end +EOS + } +} +``` + + + + + + +```hcl +{ + "kind": "proxy-defaults", + "name": "global", + "protocol": "http", + "envoy_extensions": [{ + "name": "builtin/lua", + "arguments": { + "proxy_type": "connect-proxy", + "listener": "inbound", + "script": "function envoy_on_request(request_handle)\nmeta = request_handle:streamInfo():dynamicMetadata()\nm = \nmeta:get("consul")\nrequest_handle:headers():add("x-consul-service", m["service"])\nrequest_handle:headers():add("x-consul-namespace", m["namespace"])\nrequest_handle:headers():add("x-consul-datacenter", m["datacenter"])\nrequest_handle:headers():add("x-consul-trust-domain", m["trust-domain"])\nend" + } + }] +} +``` + + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ProxyDefaults +metadata: + name: global +spec: + protocol: http + envoyExtensions: + name = "builtin/lua" + arguments: + proxyType: "connect-proxy" + listener: "inbound" + script: |- +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-service", m["service"]) + request_handle:headers():add("x-consul-namespace", m["namespace"]) + request_handle:headers():add("x-consul-datacenter", m["datacenter"]) + request_handle:headers():add("x-consul-trust-domain", m["trust-domain"]) +end +``` + + + + + +For a full list of parameters for `EnvoyExtensions`, refer to the [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) and [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries reference documentation. + +!> **Warning:** Applying `EnvoyExtensions` to `ProxyDefaults` may produce unintended consequences. We recommend enabling `EnvoyExtensions` with `ServiceDefaults` in most cases. + +Refer to [Configuration specification](#configuration-specification) section to find a full list of arguments for the Lua Envoy extension. + +### Apply the configuration entry + +Apply the `proxy-defaults` or `service-defaults` configuration entry. + + + + +```shell-session +$ consul config write lua-envoy-extension-proxy-defaults.hcl +``` + + + + +```shell-session +$ consul config write lua-envoy-extension-proxy-defaults.json + +``` + + + + +```shell-session +$ kubectl apply lua-envoy-extension-proxy-defaults.yaml +``` + + + + +## Examples + +In the following example, the `service-defaults` configure the Lua Envoy extension to insert the HTTP Lua filter for service `myservice` and add the Consul service name to the`x-consul-service` header for all inbound requests. The `ListenerType` makes it so that the extension applies only on the inbound listener of the service's connect proxy. + + + +```hcl +Kind = "service-defaults" +Name = "myservice" +EnvoyExtensions { + Name = "builtin/lua" + + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = < + +Alternatively, you can apply the same extension configuration to [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 44dff90e2f1e..23cc2d5d1aa0 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -430,6 +430,28 @@ "title": "Envoy", "path": "connect/proxies/envoy" }, + { + "title": "Envoy Extensions", + "routes": [ + { + "title": "Overview", + "path": "connect/proxies/envoy-extensions" + }, + { + "title": "Usage", + "routes": [ + { + "title": "Run Lua scripts in Envoy proxies", + "path": "connect/proxies/envoy-extensions/usage/lua" + }, + { + "title": "Invoke Lambda functions in Envoy proxies", + "path": "connect/proxies/envoy-extensions/usage/lambda" + } + ] + } + ] + }, { "title": "Built-in Proxy", "path": "connect/proxies/built-in" From b457e8bbf41d9de7337e5b0fd9a9c9255e20a6af Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 11:17:32 -0800 Subject: [PATCH 052/421] backport of commit 9fa59888faa9003489dccd6853137335cf8574d7 (#16440) Co-authored-by: cskh --- .../resolver_default_subset_test.go | 5 +- .../test/upgrade/peering_http_test.go | 110 +++++++++++++++++- 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go index 059ea39ae888..182c1cc12783 100644 --- a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go @@ -11,7 +11,6 @@ import ( libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" libutils "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" @@ -38,11 +37,11 @@ func TestTrafficManagement_ServiceResolverDefaultSubset(t *testing.T) { tcs := []testcase{ { oldversion: "1.13", - targetVersion: utils.TargetVersion, + targetVersion: libutils.TargetVersion, }, { oldversion: "1.14", - targetVersion: utils.TargetVersion, + targetVersion: libutils.TargetVersion, }, } diff --git a/test/integration/consul-container/test/upgrade/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering_http_test.go index e799cf59a8ac..f1d52777a7d6 100644 --- a/test/integration/consul-container/test/upgrade/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering_http_test.go @@ -129,7 +129,9 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { return nil, nil, nil, err } - clientConnectProxy, err := createAndRegisterStaticClientSidecarWithSplittingUpstreams(dialing) + clientConnectProxy, err := createAndRegisterStaticClientSidecarWith2Upstreams(dialing, + []string{"split-static-server", "peer-static-server"}, + ) if err != nil { return nil, nil, nil, fmt.Errorf("error creating client connect proxy in cluster %s", dialing.NetworkName) } @@ -218,6 +220,100 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { }, extraAssertion: func(clientUpstreamPort int) {}, }, + { + oldversion: "1.14", + targetVersion: utils.TargetVersion, + name: "http resolver and failover", + // Verify resolver and failover can direct traffic to server in peered cluster + // In addtional to the basic topology, this case provisions the following + // services in the dialing cluster: + // + // - a new static-client at server_0 that has two upstreams: static-server (5000) + // and peer-static-server (5001) + // - a local static-server service at client_0 + // - service-resolved named static-server with failover to static-server in accepting cluster + // - service-resolved named peer-static-server to static-server in accepting cluster + create: func(accepting *cluster.Cluster, dialing *cluster.Cluster) (libservice.Service, libservice.Service, func(), error) { + err := dialing.ConfigEntryWrite(&api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: "global", + Config: map[string]interface{}{ + "protocol": "http", + }, + }) + if err != nil { + return nil, nil, nil, err + } + + clientConnectProxy, err := createAndRegisterStaticClientSidecarWith2Upstreams(dialing, + []string{"static-server", "peer-static-server"}, + ) + if err != nil { + return nil, nil, nil, fmt.Errorf("error creating client connect proxy in cluster %s", dialing.NetworkName) + } + + // make a resolver for service peer-static-server + resolverConfigEntry := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: "peer-static-server", + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServerServiceName, + Peer: libtopology.DialingPeerName, + }, + } + err = dialing.ConfigEntryWrite(resolverConfigEntry) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing resolver config entry for %s", resolverConfigEntry.Name) + } + + // make a resolver for service static-server + resolverConfigEntry = &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: "static-server", + Failover: map[string]api.ServiceResolverFailover{ + "*": { + Targets: []api.ServiceResolverFailoverTarget{ + { + Peer: libtopology.DialingPeerName, + }, + }, + }, + }, + } + err = dialing.ConfigEntryWrite(resolverConfigEntry) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing resolver config entry for %s", resolverConfigEntry.Name) + } + + // Make a static-server in dialing cluster + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server-dialing", + HTTPPort: 8081, + GRPCPort: 8078, + } + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts) + libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName) + if err != nil { + return nil, nil, nil, err + } + + _, appPorts := clientConnectProxy.GetAddrs() + assertionFn := func() { + // assert traffic can fail-over to static-server in peered cluster and restor to local static-server + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing") + require.NoError(t, serverConnectProxy.Stop()) + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server") + require.NoError(t, serverConnectProxy.Start()) + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing") + + // assert peer-static-server resolves to static-server in peered cluster + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[1]), "static-server") + } + return serverConnectProxy, clientConnectProxy, assertionFn, nil + }, + extraAssertion: func(clientUpstreamPort int) {}, + }, } run := func(t *testing.T, tc testcase) { @@ -239,6 +335,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { _, appPort := dialing.Container.GetAddr() _, secondClientProxy, assertionAdditionalResources, err := tc.create(acceptingCluster, dialingCluster) require.NoError(t, err) + assertionAdditionalResources() tc.extraAssertion(appPort) // Upgrade the accepting cluster and assert peering is still ACTIVE @@ -294,9 +391,10 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { } } -// createAndRegisterStaticClientSidecarWithSplittingUpstreams creates a static-client-1 that -// has two upstreams: split-static-server (5000) and peer-static-server (5001) -func createAndRegisterStaticClientSidecarWithSplittingUpstreams(c *cluster.Cluster) (*libservice.ConnectContainer, error) { +// createAndRegisterStaticClientSidecarWith2Upstreams creates a static-client that +// has two upstreams connecting to destinationNames: local bind addresses are 5000 +// and 5001. +func createAndRegisterStaticClientSidecarWith2Upstreams(c *cluster.Cluster, destinationNames []string) (*libservice.ConnectContainer, error) { // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. var deferClean utils.ResettableDefer @@ -315,7 +413,7 @@ func createAndRegisterStaticClientSidecarWithSplittingUpstreams(c *cluster.Clust Proxy: &api.AgentServiceConnectProxyConfig{ Upstreams: []api.Upstream{ { - DestinationName: "split-static-server", + DestinationName: destinationNames[0], LocalBindAddress: "0.0.0.0", LocalBindPort: cluster.ServiceUpstreamLocalBindPort, MeshGateway: api.MeshGatewayConfig{ @@ -323,7 +421,7 @@ func createAndRegisterStaticClientSidecarWithSplittingUpstreams(c *cluster.Clust }, }, { - DestinationName: "peer-static-server", + DestinationName: destinationNames[1], LocalBindAddress: "0.0.0.0", LocalBindPort: cluster.ServiceUpstreamLocalBindPort2, MeshGateway: api.MeshGatewayConfig{ From 291690bbb316519ed3fecc61e4fa23592732d11e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 11:28:51 -0800 Subject: [PATCH 053/421] backport of commit e17298b6b82dd1a19e763d1964854f8c87cd9e60 (#16441) Co-authored-by: Anita Akaeze --- .../libs/cluster/container.go | 2 + .../resolver_subset_redirect_test.go | 230 ++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index bd4416a35bf4..5aa3f023eebf 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -518,10 +518,12 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR "8079/tcp", // Envoy App Listener - grpc port used by static-server "8078/tcp", // Envoy App Listener - grpc port used by static-server-v1 "8077/tcp", // Envoy App Listener - grpc port used by static-server-v2 + "8076/tcp", // Envoy App Listener - grpc port used by static-server-v3 "8080/tcp", // Envoy App Listener - http port used by static-server "8081/tcp", // Envoy App Listener - http port used by static-server-v1 "8082/tcp", // Envoy App Listener - http port used by static-server-v2 + "8083/tcp", // Envoy App Listener - http port used by static-server-v3 "9998/tcp", // Envoy App Listener "9999/tcp", // Envoy App Listener }, diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go new file mode 100644 index 000000000000..3ba808057ac9 --- /dev/null +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go @@ -0,0 +1,230 @@ +package upgrade + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/require" + "gotest.tools/assert" +) + +// TestTrafficManagement_ServiceResolverSubsetRedirect Summary +// This test starts up 4 servers and 1 client in the same datacenter. +// +// Steps: +// - Create a single agent cluster. +// - Create 2 static-servers, 2 subset servers and 1 client and sidecars for all services, then register them with Consul +// - Validate traffic is successfully redirected from server 1 to sever2-v2 as defined in the service resolver +func TestTrafficManagement_ServiceResolverSubsetRedirect(t *testing.T) { + t.Parallel() + + type testcase struct { + oldversion string + targetVersion string + } + tcs := []testcase{ + { + oldversion: "1.13", + targetVersion: utils.TargetVersion, + }, + { + oldversion: "1.14", + targetVersion: utils.TargetVersion, + }, + } + + run := func(t *testing.T, tc testcase) { + buildOpts := &libcluster.BuildOptions{ + ConsulVersion: tc.oldversion, + Datacenter: "dc1", + InjectAutoEncryption: true, + } + // If version < 1.14 disable AutoEncryption + oldVersion, _ := version.NewVersion(tc.oldversion) + if oldVersion.LessThan(utils.Version_1_14) { + buildOpts.InjectAutoEncryption = false + } + cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) + node := cluster.Agents[0] + + // Register static-server service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServer2ServiceName, + Subsets: map[string]api.ServiceResolverSubset{ + "v1": { + Filter: "Service.Meta.version == v1", + }, + "v2": { + Filter: "Service.Meta.version == v2", + }, + }, + } + err := cluster.ConfigEntryWrite(serviceResolver) + require.NoError(t, err) + + // Register static-server-2 service resolver to redirect traffic + // from static-server to static-server-2-v2 + service2Resolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServer2ServiceName, + ServiceSubset: "v2", + }, + } + err = cluster.ConfigEntryWrite(service2Resolver) + require.NoError(t, err) + + // register agent services + agentServices := setupServiceAndSubsets(t, cluster) + assertionFn, proxyRestartFn := agentServices.validateAgentServices(t) + _, port := agentServices.client.GetAddr() + _, adminPort := agentServices.client.GetAdminAddr() + + // validate static-client is up and running + libassert.AssertContainerState(t, agentServices.client, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") + assertionFn() + + // Upgrade cluster, restart sidecars then begin service traffic validation + require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) + proxyRestartFn() + + libassert.AssertContainerState(t, agentServices.client, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") + assertionFn() + + // assert 3 static-server instances are healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServer2ServiceName, false, 3) + libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server-2.default", "HEALTHY", 1) + } + + for _, tc := range tcs { + t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), + func(t *testing.T) { + run(t, tc) + }) + // test sometimes fails with error: could not start or join all agents: could not add container index 0: port not found + time.Sleep(1 * time.Second) + } +} + +func (s *registeredServices) validateAgentServices(t *testing.T) (func(), func()) { + var ( + responseFormat = map[string]string{"format": "json"} + servicePort = make(map[string]int) + proxyRestartFn func() + assertionFn func() + ) + + for serviceName, proxies := range s.services { + for _, proxy := range proxies { + _, adminPort := proxy.GetAdminAddr() + servicePort[serviceName] = adminPort + } + } + + assertionFn = func() { + // validate services proxy admin is up + for serviceName, adminPort := range servicePort { + _, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode, fmt.Sprintf("%s cannot be reached %v", serviceName, statusCode)) + + // certs are valid + libassert.AssertEnvoyPresentsCertURI(t, adminPort, serviceName) + } + } + + for _, serviceConnectProxy := range s.services { + for _, proxy := range serviceConnectProxy { + proxyRestartFn = func() { require.NoErrorf(t, proxy.Restart(), "%s", proxy.GetName()) } + } + } + return assertionFn, proxyRestartFn +} + +type registeredServices struct { + client libservice.Service + services map[string][]libservice.Service +} + +// create 3 servers and 1 client +func setupServiceAndSubsets(t *testing.T, cluster *libcluster.Cluster) *registeredServices { + node := cluster.Agents[0] + client := node.GetClient() + + // create static-servers and subsets + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, + } + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + + serviceOpts2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2", + HTTPPort: 8081, + GRPCPort: 8078, + } + _, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8082, + GRPCPort: 8077, + } + _, server2ConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + serviceOptsV2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v2", + Meta: map[string]string{"version": "v2"}, + HTTPPort: 8083, + GRPCPort: 8076, + } + _, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + // Create a client proxy instance with the server as an upstream + clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + + // return a map of all services created + tmpServices := map[string][]libservice.Service{} + tmpServices[libservice.StaticClientServiceName] = append(tmpServices[libservice.StaticClientServiceName], clientConnectProxy) + tmpServices[libservice.StaticServerServiceName] = append(tmpServices[libservice.StaticServerServiceName], serverConnectProxy) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxy) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV1) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV2) + + return ®isteredServices{ + client: clientConnectProxy, + services: tmpServices, + } +} From 9902dab7bccbcca06aea596179fec0134e259d31 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 12:06:38 -0800 Subject: [PATCH 054/421] Backport of Basic gobased API gateway spinup test into release/1.15.x (#16423) * backport of commit 60a62add31254c5677df46c4e3bcc9c43af09dfb * backport of commit 560cd55675f07be83cfb2b9d51fc321863270c8f * backport of commit bd3ff146387b04e49376f0f998cb3db176d0755e * backport of commit b8a4c8319d05ae66baa64607fe86f85777a6e15c * backport of commit 184b6bbcb06164c28da27e6bc06d635796ea38c3 * backport of commit cc213b67b437c9d64f3e34c78007a7c15eb005b0 * backport of commit e706bc7cace2097f52e7d21c6b80ff579355d35f * backport of commit 064a3c0104ffeb43c0073fc575be238fef1558b4 * backport of commit b8aa15ef5453d90f5bb1f036c2ddfcb977c05007 * updated GetPort interface * fix getport * fix syntax issues * fix gofmt error * removed deprecated random calls, removed unused port variable * clean up duplicate comment * Update test/integration/consul-container/test/gateways/gateway_endpoint_test.go * parralelize this test * delete extra line * clean up unneeded comments --------- Co-authored-by: Sarah Alsmiller Co-authored-by: Andrew Stucki Co-authored-by: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> --- .../consul-container/libs/cluster/agent.go | 1 + .../consul-container/libs/cluster/cluster.go | 7 +- .../libs/cluster/container.go | 14 +- .../consul-container/libs/service/connect.go | 5 + .../consul-container/libs/service/examples.go | 24 +- .../consul-container/libs/service/gateway.go | 52 +++- .../consul-container/libs/service/helpers.go | 8 +- .../consul-container/libs/service/service.go | 2 + .../test/gateways/gateway_endpoint_test.go | 250 +++++++++++++++++ .../test/gateways/http_route_test.go | 258 ++++++++++++++++++ 10 files changed, 588 insertions(+), 33 deletions(-) create mode 100644 test/integration/consul-container/test/gateways/gateway_endpoint_test.go create mode 100644 test/integration/consul-container/test/gateways/http_route_test.go diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 24c5b37bd5f4..3c4eeafc13ce 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -22,6 +22,7 @@ type Agent interface { GetConfig() Config GetInfo() AgentInfo GetDatacenter() string + GetNetwork() string IsServer() bool RegisterTermination(func() error) Terminate() error diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index c569a21a38eb..ba24b798c76d 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -63,7 +63,7 @@ func NewN(t TestingT, conf Config, count int) (*Cluster, error) { // // The provided TestingT is used to register a cleanup function to terminate // the cluster. -func New(t TestingT, configs []Config) (*Cluster, error) { +func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) { id, err := shortid.Generate() if err != nil { return nil, fmt.Errorf("could not cluster id: %w", err) @@ -99,7 +99,7 @@ func New(t TestingT, configs []Config) (*Cluster, error) { _ = cluster.Terminate() }) - if err := cluster.Add(configs, true); err != nil { + if err := cluster.Add(configs, true, ports...); err != nil { return nil, fmt.Errorf("could not start or join all agents: %w", err) } @@ -115,7 +115,7 @@ func (c *Cluster) AddN(conf Config, count int, join bool) error { } // Add starts an agent with the given configuration and joins it with the existing cluster -func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) { +func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) { if c.Index == 0 && !serfJoin { return fmt.Errorf("the first call to Cluster.Add must have serfJoin=true") } @@ -135,6 +135,7 @@ func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) { context.Background(), conf, c, + ports..., ) if err != nil { return fmt.Errorf("could not add container index %d: %w", idx, err) diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 5aa3f023eebf..4e577462a185 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -73,7 +73,7 @@ func (c *consulContainerNode) ClaimAdminPort() (int, error) { } // NewConsulContainer starts a Consul agent in a container with the given config. -func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (Agent, error) { +func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, ports ...int) (Agent, error) { network := cluster.NetworkName index := cluster.Index if config.ScratchDir == "" { @@ -128,7 +128,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A addtionalNetworks: []string{"bridge", network}, hostname: fmt.Sprintf("agent-%d", index), } - podReq, consulReq := newContainerRequest(config, opts) + podReq, consulReq := newContainerRequest(config, opts, ports...) // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. @@ -291,6 +291,10 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A return node, nil } +func (c *consulContainerNode) GetNetwork() string { + return c.network +} + func (c *consulContainerNode) GetName() string { if c.container == nil { return c.consulReq.Name // TODO: is this safe to do all the time? @@ -501,7 +505,7 @@ type containerOpts struct { addtionalNetworks []string } -func newContainerRequest(config Config, opts containerOpts) (podRequest, consulRequest testcontainers.ContainerRequest) { +func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRequest, consulRequest testcontainers.ContainerRequest) { skipReaper := isRYUKDisabled() pod := testcontainers.ContainerRequest{ @@ -541,6 +545,10 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", basePort+i)) } + for _, port := range ports { + pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port)) + } + // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". // If we only wait until the log stream starts, subsequent commands to agents will fail. // TODO: optimize the wait strategy diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 49a340bd2a16..b7f617c0df0c 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -2,6 +2,7 @@ package service import ( "context" + "errors" "fmt" "io" "path/filepath" @@ -58,6 +59,10 @@ func (g ConnectContainer) GetAddrs() (string, []int) { return g.ip, g.appPort } +func (g ConnectContainer) GetPort(port int) (int, error) { + return 0, errors.New("not implemented") +} + func (g ConnectContainer) Restart() error { _, err := g.GetStatus() if err != nil { diff --git a/test/integration/consul-container/libs/service/examples.go b/test/integration/consul-container/libs/service/examples.go index 3d6258eaa9b3..0c28a1526251 100644 --- a/test/integration/consul-container/libs/service/examples.go +++ b/test/integration/consul-container/libs/service/examples.go @@ -68,6 +68,10 @@ func (g exampleContainer) GetAddrs() (string, []int) { return "", nil } +func (g exampleContainer) GetPort(port int) (int, error) { + return 0, nil +} + func (g exampleContainer) Restart() error { return fmt.Errorf("Restart Unimplemented by ConnectContainer") } @@ -121,7 +125,7 @@ func (c exampleContainer) GetStatus() (string, error) { return state.Status, err } -func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent) (Service, error) { +func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent, containerArgs ...string) (Service, error) { namePrefix := fmt.Sprintf("%s-service-example-%s", node.GetDatacenter(), name) containerName := utils.RandName(namePrefix) @@ -135,18 +139,22 @@ func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort grpcPortStr = strconv.Itoa(grpcPort) ) + command := []string{ + "server", + "-http-port", httpPortStr, + "-grpc-port", grpcPortStr, + "-redirect-port", "-disabled", + } + + command = append(command, containerArgs...) + req := testcontainers.ContainerRequest{ Image: hashicorpDockerProxy + "/fortio/fortio", WaitingFor: wait.ForLog("").WithStartupTimeout(10 * time.Second), AutoRemove: false, Name: containerName, - Cmd: []string{ - "server", - "-http-port", httpPortStr, - "-grpc-port", grpcPortStr, - "-redirect-port", "-disabled", - }, - Env: map[string]string{"FORTIO_NAME": name}, + Cmd: command, + Env: map[string]string{"FORTIO_NAME": name}, } info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{httpPortStr, grpcPortStr}) diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 7028a612928c..8a9b87aa1964 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -20,12 +20,13 @@ import ( // gatewayContainer type gatewayContainer struct { - ctx context.Context - container testcontainers.Container - ip string - port int - adminPort int - serviceName string + ctx context.Context + container testcontainers.Container + ip string + port int + adminPort int + serviceName string + portMappings map[int]int } var _ Service = (*gatewayContainer)(nil) @@ -105,6 +106,15 @@ func (g gatewayContainer) GetAdminAddr() (string, int) { return "localhost", g.adminPort } +func (g gatewayContainer) GetPort(port int) (int, error) { + p, ok := g.portMappings[port] + if !ok { + return 0, fmt.Errorf("port does not exist") + } + return p, nil + +} + func (g gatewayContainer) Restart() error { _, err := g.container.State(g.ctx) if err != nil { @@ -130,7 +140,7 @@ func (g gatewayContainer) GetStatus() (string, error) { return state.Status, err } -func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent) (Service, error) { +func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent, ports ...int) (Service, error) { nodeConfig := node.GetConfig() if nodeConfig.ScratchDir == "" { return nil, fmt.Errorf("node ScratchDir is required") @@ -207,21 +217,33 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl adminPortStr = strconv.Itoa(adminPort) ) - info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{ + extraPorts := []string{} + for _, port := range ports { + extraPorts = append(extraPorts, strconv.Itoa(port)) + } + + info, err := cluster.LaunchContainerOnNode(ctx, node, req, append( + extraPorts, portStr, adminPortStr, - }) + )) if err != nil { return nil, err } + portMappings := make(map[int]int) + for _, port := range ports { + portMappings[port] = info.MappedPorts[strconv.Itoa(port)].Int() + } + out := &gatewayContainer{ - ctx: ctx, - container: info.Container, - ip: info.IP, - port: info.MappedPorts[portStr].Int(), - adminPort: info.MappedPorts[adminPortStr].Int(), - serviceName: name, + ctx: ctx, + container: info.Container, + ip: info.IP, + port: info.MappedPorts[portStr].Int(), + adminPort: info.MappedPorts[adminPortStr].Int(), + serviceName: name, + portMappings: portMappings, } return out, nil diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index f7b963ff5a0a..47a8c1364255 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -35,7 +35,7 @@ type ServiceOpts struct { } // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers -func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int, svc *api.AgentServiceRegistration) (Service, Service, error) { +func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int, svc *api.AgentServiceRegistration, containerArgs ...string) (Service, Service, error) { // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. var deferClean utils.ResettableDefer @@ -46,7 +46,7 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int } // Create a service and proxy instance - serverService, err := NewExampleService(context.Background(), svc.ID, svc.Port, grpcPort, node) + serverService, err := NewExampleService(context.Background(), svc.ID, svc.Port, grpcPort, node, containerArgs...) if err != nil { return nil, nil, err } @@ -68,7 +68,7 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int return serverService, serverConnectProxy, nil } -func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { +func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts, containerArgs ...string) (Service, Service, error) { // Register the static-server service and sidecar first to prevent race with sidecar // trying to get xDS before it's ready req := &api.AgentServiceRegistration{ @@ -88,7 +88,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts }, Meta: serviceOpts.Meta, } - return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req) + return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req, containerArgs...) } func CreateAndRegisterStaticServerAndSidecarWithChecks(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { diff --git a/test/integration/consul-container/libs/service/service.go b/test/integration/consul-container/libs/service/service.go index f32bd67ff8b2..f4efa4f909c4 100644 --- a/test/integration/consul-container/libs/service/service.go +++ b/test/integration/consul-container/libs/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "github.com/hashicorp/consul/api" ) @@ -13,6 +14,7 @@ type Service interface { Export(partition, peer string, client *api.Client) error GetAddr() (string, int) GetAddrs() (string, []int) + GetPort(port int) (int, error) // GetAdminAddr returns the external admin address GetAdminAddr() (string, int) GetLogs() (string, error) diff --git a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go new file mode 100644 index 000000000000..e750a5b1fdc3 --- /dev/null +++ b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go @@ -0,0 +1,250 @@ +package gateways + +import ( + "context" + "fmt" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/hashicorp/go-cleanhttp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io" + "net/http" + "strings" + "testing" + "time" +) + +var ( + checkTimeout = 1 * time.Minute + checkInterval = 1 * time.Second +) + +// Creates a gateway service and tests to see if it is routable +func TestAPIGatewayCreate(t *testing.T) { + t.Skip() + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + listenerPortOne := 6000 + + cluster := createCluster(t, listenerPortOne) + + client := cluster.APIClient(0) + + //setup + apiGateway := &api.APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: "api-gateway", + Listeners: []api.APIGatewayListener{ + { + Port: listenerPortOne, + Protocol: "tcp", + }, + }, + } + _, _, err := client.ConfigEntries().Set(apiGateway, nil) + assert.NoError(t, err) + + tcpRoute := &api.TCPRouteConfigEntry{ + Kind: "tcp-route", + Name: "api-gateway-route", + Parents: []api.ResourceReference{ + { + Kind: "api-gateway", + Name: "api-gateway", + }, + }, + Services: []api.TCPService{ + { + Name: libservice.StaticServerServiceName, + }, + }, + } + + _, _, err = client.ConfigEntries().Set(tcpRoute, nil) + assert.NoError(t, err) + + // Create a client proxy instance with the server as an upstream + _, gatewayService := createServices(t, cluster, listenerPortOne) + + //check statuses + gatewayReady := false + routeReady := false + + //make sure the gateway/route come online + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get("api-gateway", "api-gateway", nil) + assert.NoError(t, err) + apiEntry := entry.(*api.APIGatewayConfigEntry) + gatewayReady = isAccepted(apiEntry.Status.Conditions) + + e, _, err := client.ConfigEntries().Get("tcp-route", "api-gateway-route", nil) + assert.NoError(t, err) + routeEntry := e.(*api.TCPRouteConfigEntry) + routeReady = isBound(routeEntry.Status.Conditions) + + return gatewayReady && routeReady + }, time.Second*10, time.Second*1) + + port, err := gatewayService.GetPort(listenerPortOne) + assert.NoError(t, err) + libassert.HTTPServiceEchoes(t, "localhost", port, "") +} + +func isAccepted(conditions []api.Condition) bool { + return conditionStatusIsValue("Accepted", "True", conditions) +} + +func isBound(conditions []api.Condition) bool { + return conditionStatusIsValue("Bound", "True", conditions) +} + +func conditionStatusIsValue(typeName string, statusValue string, conditions []api.Condition) bool { + for _, c := range conditions { + if c.Type == typeName && c.Status == statusValue { + return true + } + } + return false +} + +// TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some +func createCluster(t *testing.T, ports ...int) *libcluster.Cluster { + opts := libcluster.BuildOptions{ + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + } + ctx := libcluster.NewBuildContext(t, opts) + + conf := libcluster.NewConfigBuilder(ctx). + ToAgentConfig(t) + t.Logf("Cluster config:\n%s", conf.JSON) + + configs := []libcluster.Config{*conf} + + cluster, err := libcluster.New(t, configs, ports...) + require.NoError(t, err) + + node := cluster.Agents[0] + client := node.GetClient() + + libcluster.WaitForLeader(t, cluster, client) + libcluster.WaitForMembers(t, client, 1) + + // Default Proxy Settings + ok, err := utils.ApplyDefaultProxySettings(client) + require.NoError(t, err) + require.True(t, ok) + + require.NoError(t, err) + + return cluster +} + +func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service { + node := cluster.Agents[0] + client := node.GetClient() + // Create a service and proxy instance + service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...) + assert.NoError(t, err) + + libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy") + libassert.CatalogServiceExists(t, client, serviceOpts.Name) + + return service + +} +func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) { + node := cluster.Agents[0] + client := node.GetClient() + // Create a service and proxy instance + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, + } + + clientConnectProxy := createService(t, cluster, serviceOpts, nil) + + gatewayService, err := libservice.NewGatewayService(context.Background(), "api-gateway", "api", cluster.Agents[0], ports...) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "api-gateway") + + return clientConnectProxy, gatewayService +} + +// checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances + +type checkOptions struct { + debug bool + statusCode int + testName string +} + +func checkRoute(t *testing.T, ip string, port int, path string, headers map[string]string, expected checkOptions) { + const phrase = "hello" + + failer := func() *retry.Timer { + return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} + } + + client := cleanhttp.DefaultClient() + url := fmt.Sprintf("http://%s:%d", ip, port) + + if path != "" { + url += "/" + path + } + + retry.RunWith(failer(), t, func(r *retry.R) { + t.Logf("making call to %s", url) + reader := strings.NewReader(phrase) + req, err := http.NewRequest("POST", url, reader) + assert.NoError(t, err) + headers["content-type"] = "text/plain" + + for k, v := range headers { + req.Header.Set(k, v) + + if k == "Host" { + req.Host = v + } + } + res, err := client.Do(req) + if err != nil { + t.Log(err) + r.Fatal("could not make call to service ", url) + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + r.Fatal("could not read response body ", url) + } + + assert.Equal(t, expected.statusCode, res.StatusCode) + if expected.statusCode != res.StatusCode { + r.Fatal("unexpected response code returned") + } + + //if debug is expected, debug should be in the response body + assert.Equal(t, expected.debug, strings.Contains(string(body), "debug")) + if expected.statusCode != res.StatusCode { + r.Fatal("unexpected response body returned") + } + + if !strings.Contains(string(body), phrase) { + r.Fatal("received an incorrect response ", string(body)) + } + + }) +} diff --git a/test/integration/consul-container/test/gateways/http_route_test.go b/test/integration/consul-container/test/gateways/http_route_test.go new file mode 100644 index 000000000000..26f83ae92eca --- /dev/null +++ b/test/integration/consul-container/test/gateways/http_route_test.go @@ -0,0 +1,258 @@ +package gateways + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func getNamespace() string { + return "" +} + +// randomName generates a random name of n length with the provided +// prefix. If prefix is omitted, the then entire name is random char. +func randomName(prefix string, n int) string { + if n == 0 { + n = 32 + } + if len(prefix) >= n { + return prefix + } + p := make([]byte, n) + rand.Read(p) + return fmt.Sprintf("%s-%s", prefix, hex.EncodeToString(p))[:n] +} + +func TestHTTPRouteFlattening(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + //infrastructure set up + listenerPort := 6000 + //create cluster + cluster := createCluster(t, listenerPort) + client := cluster.Agents[0].GetClient() + service1ResponseCode := 200 + service2ResponseCode := 418 + serviceOne := createService(t, cluster, &libservice.ServiceOpts{ + Name: "service1", + ID: "service1", + HTTPPort: 8080, + GRPCPort: 8079, + }, []string{ + //customizes response code so we can distinguish between which service is responding + "-echo-server-default-params", fmt.Sprintf("status=%d", service1ResponseCode), + }) + serviceTwo := createService(t, cluster, &libservice.ServiceOpts{ + Name: "service2", + ID: "service2", + HTTPPort: 8081, + GRPCPort: 8082, + }, []string{ + "-echo-server-default-params", fmt.Sprintf("status=%d", service2ResponseCode), + }, + ) + + //TODO this should only matter in consul enterprise I believe? + namespace := getNamespace() + gatewayName := randomName("gw", 16) + routeOneName := randomName("route", 16) + routeTwoName := randomName("route", 16) + path1 := "/" + path2 := "/v2" + + //write config entries + proxyDefaults := &api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, + Namespace: namespace, + Config: map[string]interface{}{ + "protocol": "http", + }, + } + + _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) + assert.NoError(t, err) + + apiGateway := &api.APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: gatewayName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerPort, + Protocol: "http", + }, + }, + } + + routeOne := &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: routeOneName, + Parents: []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayName, + Namespace: namespace, + }, + }, + Hostnames: []string{ + "test.foo", + "test.example", + }, + Namespace: namespace, + Rules: []api.HTTPRouteRule{ + { + Services: []api.HTTPService{ + { + Name: serviceOne.GetServiceName(), + Namespace: namespace, + }, + }, + Matches: []api.HTTPMatch{ + { + Path: api.HTTPPathMatch{ + Match: api.HTTPPathMatchPrefix, + Value: path1, + }, + }, + }, + }, + }, + } + + routeTwo := &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: routeTwoName, + Parents: []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayName, + Namespace: namespace, + }, + }, + Hostnames: []string{ + "test.foo", + }, + Namespace: namespace, + Rules: []api.HTTPRouteRule{ + { + Services: []api.HTTPService{ + { + Name: serviceTwo.GetServiceName(), + Namespace: namespace, + }, + }, + Matches: []api.HTTPMatch{ + { + Path: api.HTTPPathMatch{ + Match: api.HTTPPathMatchPrefix, + Value: path2, + }, + }, + { + Headers: []api.HTTPHeaderMatch{{ + Match: api.HTTPHeaderMatchExact, + Name: "x-v2", + Value: "v2", + }}, + }, + }, + }, + }, + } + + _, _, err = client.ConfigEntries().Set(apiGateway, nil) + assert.NoError(t, err) + _, _, err = client.ConfigEntries().Set(routeOne, nil) + assert.NoError(t, err) + _, _, err = client.ConfigEntries().Set(routeTwo, nil) + assert.NoError(t, err) + + //create gateway service + gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, gatewayName) + + //make sure config entries have been properly created + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace}) + assert.NoError(t, err) + if entry == nil { + return false + } + apiEntry := entry.(*api.APIGatewayConfigEntry) + t.Log(entry) + return isAccepted(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) + + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeOneName, &api.QueryOptions{Namespace: namespace}) + assert.NoError(t, err) + if entry == nil { + return false + } + + apiEntry := entry.(*api.HTTPRouteConfigEntry) + t.Log(entry) + return isBound(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) + + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeTwoName, nil) + assert.NoError(t, err) + if entry == nil { + return false + } + + apiEntry := entry.(*api.HTTPRouteConfigEntry) + return isBound(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) + + //gateway resolves routes + ip := "localhost" + gatewayPort, err := gatewayService.GetPort(listenerPort) + assert.NoError(t, err) + + //Same v2 path with and without header + checkRoute(t, ip, gatewayPort, "v2", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"}) + checkRoute(t, ip, gatewayPort, "v2", map[string]string{ + "Host": "test.foo", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"}) + + ////v1 path with the header + checkRoute(t, ip, gatewayPort, "check", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"}) + + checkRoute(t, ip, gatewayPort, "v2/path/value", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"}) + + //hit service 1 by hitting root path + checkRoute(t, ip, gatewayPort, "", map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"}) + + //hit service 1 by hitting v2 path with v1 hostname + checkRoute(t, ip, gatewayPort, "v2", map[string]string{ + "Host": "test.example", + }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"}) + +} From 5af37749b84e1338464f8961ce6a4ea76ebd9c93 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 15:47:25 -0800 Subject: [PATCH 055/421] Backport of UI: Fix rendering issue in search and lists into release/1.15.x (#16446) * backport of commit 27fc0f332643b83008e57e0e2cd33c8aa8ff577a * backport of commit 5ef3d2a4367d29aeda8dca17c82bac38a55e6300 --------- Co-authored-by: wenincode --- .changelog/16444.txt | 3 + ui/packages/consul-ui/package.json | 2 +- ui/yarn.lock | 146 +++++++++++++++++++++++++++-- 3 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 .changelog/16444.txt diff --git a/.changelog/16444.txt b/.changelog/16444.txt new file mode 100644 index 000000000000..542f0560fecf --- /dev/null +++ b/.changelog/16444.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: Fix issue with lists and filters not rendering properly +``` diff --git a/ui/packages/consul-ui/package.json b/ui/packages/consul-ui/package.json index 91467dd8f4fd..445d12235dfc 100644 --- a/ui/packages/consul-ui/package.json +++ b/ui/packages/consul-ui/package.json @@ -120,7 +120,7 @@ "ember-cli-page-object": "^1.17.11", "ember-cli-postcss": "^8.1.0", "ember-cli-sri": "^2.1.1", - "ember-cli-string-helpers": "^5.0.0", + "ember-cli-string-helpers": "^6.1.0", "ember-cli-template-lint": "^2.0.1", "ember-cli-terser": "^4.0.2", "ember-cli-yadda": "^0.7.0", diff --git a/ui/yarn.lock b/ui/yarn.lock index 96953879f60a..6eee746a73ea 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"@ampproject/remapping@^2.1.0": +"@ampproject/remapping@^2.1.0", "@ampproject/remapping@^2.2.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.0.tgz#56c133824780de3174aed5ab6834f3026790154d" integrity sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w== @@ -58,6 +58,11 @@ resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.1.tgz#f2e6ef7790d8c8dbf03d379502dcc246dcce0b30" integrity sha512-EWZ4mE2diW3QALKvDMiXnbZpRvlj+nayZ112nK93SnhqOtpdsbVD4W+2tEoT3YNBAG9RBR0ISY758ZkOgsn6pQ== +"@babel/compat-data@^7.20.5": + version "7.21.0" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.21.0.tgz#c241dc454e5b5917e40d37e525e2f4530c399298" + integrity sha512-gMuZsmsgxk/ENC3O/fRw5QY8A9/uxQbbCEypnLIiYYc/qVJtEV7ouxC3EllIIwNzMqAQee5tanFabWsUOutS7g== + "@babel/core@^7.0.0", "@babel/core@^7.1.6", "@babel/core@^7.12.0", "@babel/core@^7.12.3", "@babel/core@^7.2.2", "@babel/core@^7.3.4", "@babel/core@^7.7.5": version "7.13.10" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.13.10.tgz#07de050bbd8193fcd8a3c27918c0890613a94559" @@ -80,6 +85,27 @@ semver "^6.3.0" source-map "^0.5.0" +"@babel/core@^7.13.10": + version "7.21.0" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.21.0.tgz#1341aefdcc14ccc7553fcc688dd8986a2daffc13" + integrity sha512-PuxUbxcW6ZYe656yL3EAhpy7qXKq0DmYsrJLpbB8XrsCP9Nm+XCg9XFMb5vIDliPD7+U/+M+QJlH17XOcB7eXA== + dependencies: + "@ampproject/remapping" "^2.2.0" + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.21.0" + "@babel/helper-compilation-targets" "^7.20.7" + "@babel/helper-module-transforms" "^7.21.0" + "@babel/helpers" "^7.21.0" + "@babel/parser" "^7.21.0" + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.21.0" + "@babel/types" "^7.21.0" + convert-source-map "^1.7.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.2" + semver "^6.3.0" + "@babel/core@^7.13.8": version "7.20.2" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.2.tgz#8dc9b1620a673f92d3624bd926dc49a52cf25b92" @@ -158,6 +184,16 @@ "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" +"@babel/generator@^7.21.0", "@babel/generator@^7.21.1": + version "7.21.1" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.21.1.tgz#951cc626057bc0af2c35cd23e9c64d384dea83dd" + integrity sha512-1lT45bAYlQhFn/BHivJs43AiW2rg3/UbLyShGfF3C0KmHvO5fSghWd5kBJy30kpRRucGzXStvnnCFniCR2kXAA== + dependencies: + "@babel/types" "^7.21.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.13.tgz#0f58e86dfc4bb3b1fcd7db806570e177d439b6ab" @@ -251,6 +287,17 @@ browserslist "^4.21.3" semver "^6.3.0" +"@babel/helper-compilation-targets@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz#a6cd33e93629f5eb473b021aac05df62c4cd09bb" + integrity sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ== + dependencies: + "@babel/compat-data" "^7.20.5" + "@babel/helper-validator-option" "^7.18.6" + browserslist "^4.21.3" + lru-cache "^5.1.1" + semver "^6.3.0" + "@babel/helper-create-class-features-plugin@^7.13.0", "@babel/helper-create-class-features-plugin@^7.5.5", "@babel/helper-create-class-features-plugin@^7.8.3": version "7.13.11" resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.13.11.tgz#30d30a005bca2c953f5653fc25091a492177f4f6" @@ -419,6 +466,14 @@ "@babel/template" "^7.18.10" "@babel/types" "^7.19.0" +"@babel/helper-function-name@^7.21.0": + version "7.21.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.21.0.tgz#d552829b10ea9f120969304023cd0645fa00b1b4" + integrity sha512-HfK1aMRanKHpxemaY2gqBmL04iAPOPRj7DxtNbiDOrJK+gdwkiNRVpCpUJYbUT+aZyemKN8brqTOxzCaG6ExRg== + dependencies: + "@babel/template" "^7.20.7" + "@babel/types" "^7.21.0" + "@babel/helper-get-function-arity@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583" @@ -554,6 +609,20 @@ "@babel/traverse" "^7.20.1" "@babel/types" "^7.20.2" +"@babel/helper-module-transforms@^7.21.0": + version "7.21.2" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.21.2.tgz#160caafa4978ac8c00ac66636cb0fa37b024e2d2" + integrity sha512-79yj2AR4U/Oqq/WOV7Lx6hUjau1Zfo4cI+JLAVYeMV5XIlbOhmjEk5ulbTc9fMpmlojzZHkUUxAiK+UKn+hNQQ== + dependencies: + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-module-imports" "^7.18.6" + "@babel/helper-simple-access" "^7.20.2" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/helper-validator-identifier" "^7.19.1" + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.21.2" + "@babel/types" "^7.21.2" + "@babel/helper-optimise-call-expression@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.13.tgz#5c02d171b4c8615b1e7163f888c1c81c30a2aaea" @@ -822,6 +891,15 @@ "@babel/traverse" "^7.20.1" "@babel/types" "^7.20.0" +"@babel/helpers@^7.21.0": + version "7.21.0" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.21.0.tgz#9dd184fb5599862037917cdc9eecb84577dc4e7e" + integrity sha512-XXve0CBtOW0pd7MRzzmoyuSj0e3SEzj8pgyFxnTT1NJZL38BD1MK7yYrm8yefRPIDvNNe14xR4FdbHwpInD4rA== + dependencies: + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.21.0" + "@babel/types" "^7.21.0" + "@babel/highlight@^7.10.4", "@babel/highlight@^7.12.13": version "7.13.10" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1" @@ -869,6 +947,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.2.tgz#9aeb9b92f64412b5f81064d46f6a1ac0881337f4" integrity sha512-afk318kh2uKbo7BEj2QtEi8HVCGrwHUffrYDy7dgVcSa2j9lY3LDjPzcyGdpX7xgm35aWqvciZJ4WKmdF/SxYg== +"@babel/parser@^7.20.7", "@babel/parser@^7.21.0", "@babel/parser@^7.21.2": + version "7.21.2" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.2.tgz#dacafadfc6d7654c3051a66d6fe55b6cb2f2a0b3" + integrity sha512-URpaIJQwEkEC2T9Kn+Ai6Xe/02iNaVCuT/PtoRz3GPVJVDpPd7mLo+VddTbhCRU9TXqW5mSrQfXZyi8kDKOVpQ== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.16.7.tgz#4eda6d6c2a0aa79c70fa7b6da67763dfe2141050" @@ -2530,6 +2613,15 @@ "@babel/parser" "^7.18.10" "@babel/types" "^7.18.10" +"@babel/template@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.20.7.tgz#a15090c2839a83b02aa996c0b4994005841fd5a8" + integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@babel/traverse@^7.1.6", "@babel/traverse@^7.12.1", "@babel/traverse@^7.13.0", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.13.0.tgz#6d95752475f86ee7ded06536de309a65fc8966cc" @@ -2593,6 +2685,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.21.0", "@babel/traverse@^7.21.2": + version "7.21.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.2.tgz#ac7e1f27658750892e815e60ae90f382a46d8e75" + integrity sha512-ts5FFU/dSUPS13tv8XiEObDu9K+iagEKME9kAbaP7r0Y9KtZJZ+NGndDvWoRAYNpeWafbpFeki3q9QoMD6gxyw== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.21.1" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.21.0" + "@babel/helper-hoist-variables" "^7.18.6" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/parser" "^7.21.2" + "@babel/types" "^7.21.2" + debug "^4.1.0" + globals "^11.1.0" + "@babel/types@^7.1.6", "@babel/types@^7.12.1", "@babel/types@^7.12.13", "@babel/types@^7.13.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0", "@babel/types@^7.7.2": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.0.tgz#74424d2816f0171b4100f0ab34e9a374efdf7f80" @@ -2628,6 +2736,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.21.2": + version "7.21.2" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.21.2.tgz#92246f6e00f91755893c2876ad653db70c8310d1" + integrity sha512-3wRZSs7jiFaB8AjxiiD+VqN5DTG2iRvJGQ+qYFrs/654lg6kGTQWIOFjlBo5RaXuAZjBmP3+OQH4dmhqiiyYxw== + dependencies: + "@babel/helper-string-parser" "^7.19.4" + "@babel/helper-validator-identifier" "^7.19.1" + to-fast-properties "^2.0.0" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -3524,7 +3641,7 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" -"@jridgewell/resolve-uri@^3.0.3": +"@jridgewell/resolve-uri@3.1.0", "@jridgewell/resolve-uri@^3.0.3": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== @@ -3542,7 +3659,7 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@jridgewell/sourcemap-codec@^1.4.10": +"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== @@ -3555,6 +3672,14 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" +"@jridgewell/trace-mapping@^0.3.17": + version "0.3.17" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" + integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== + dependencies: + "@jridgewell/resolve-uri" "3.1.0" + "@jridgewell/sourcemap-codec" "1.4.14" + "@lit/reactive-element@^1.2.1": version "1.2.1" resolved "https://registry.yarnpkg.com/@lit/reactive-element/-/reactive-element-1.2.1.tgz#8620d7f0baf63e12821fa93c34d21e23477736f7" @@ -8404,13 +8529,15 @@ ember-cli-sri@^2.1.1: dependencies: broccoli-sri-hash "^2.1.0" -ember-cli-string-helpers@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ember-cli-string-helpers/-/ember-cli-string-helpers-5.0.0.tgz#b1e08ec3ca1c9a457f9fd9aafff60b5939fbf91d" - integrity sha512-PodD3Uf7BkOXIu95E6cWEC0ERroTiUOAwOr828Vb+fPFtV7WYNSC27C9Ds1ggCyyRqnXmpS0JSqCTN1gPZfkWQ== +ember-cli-string-helpers@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/ember-cli-string-helpers/-/ember-cli-string-helpers-6.1.0.tgz#aeb96112bb91c540b869ed8b9c680f7fd5859cb6" + integrity sha512-Lw8B6MJx2n8CNF2TSIKs+hWLw0FqSYjr2/NRPyquyYA05qsl137WJSYW3ZqTsLgoinHat0DGF2qaCXocLhLmyA== dependencies: + "@babel/core" "^7.13.10" broccoli-funnel "^3.0.3" ember-cli-babel "^7.7.3" + resolve "^1.20.0" ember-cli-string-utils@^1.0.0, ember-cli-string-utils@^1.1.0: version "1.1.0" @@ -12212,6 +12339,11 @@ json5@^2.2.1: resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== +json5@^2.2.2: + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + jsonfile@^2.1.0: version "2.4.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-2.4.0.tgz#3736a2b428b87bbda0cc83b53fa3d633a35c2ae8" From acb0c3bbf6d9b932de013ae84f155c435cc85a4b Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Feb 2023 17:31:34 -0800 Subject: [PATCH 056/421] Backport of Update docs for consul-k8s 1.1.0 into release/1.15.x (#16448) * manual cherry pick --------- Co-authored-by: Curt Bushko --- .../content/docs/connect/proxies/envoy.mdx | 12 +- website/content/docs/k8s/compatibility.mdx | 2 +- website/content/docs/k8s/helm.mdx | 196 +++++++++--------- 3 files changed, 99 insertions(+), 111 deletions(-) diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 19e98a136765..3186b645c2f8 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -39,20 +39,18 @@ Consul supports **four major Envoy releases** at the beginning of each major Con | Consul Version | Compatible Envoy Versions | | ------------------- | -----------------------------------------------------------------------------------| +| 1.15.x | 1.25.1, 1.24.2, 1.23.4, 1.22.5 | | 1.14.x | 1.24.0, 1.23.1, 1.22.5, 1.21.5 | | 1.13.x | 1.23.1, 1.22.5, 1.21.5, 1.20.7 | -| 1.12.x | 1.22.5, 1.21.5, 1.20.7, 1.19.5 | - -1. Envoy 1.20.1 and earlier are vulnerable to [CVE-2022-21654](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21654) and [CVE-2022-21655](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21655). Both CVEs were patched in Envoy versions 1.18.6, 1.19.3, and 1.20.2. -Envoy 1.16.x and older releases are no longer supported (see [HCSEC-2022-07](https://discuss.hashicorp.com/t/hcsec-2022-07-consul-s-connect-service-mesh-affected-by-recent-envoy-security-releases/36332)). Consul 1.9.x clusters should be upgraded to 1.10.x and Envoy upgraded to the latest supported Envoy version for that release, 1.18.6. ### Envoy and Consul Dataplane Consul Dataplane is a feature introduced in Consul v1.14. Because each version of Consul Dataplane supports one specific version of Envoy, you must use the following versions of Consul, Consul Dataplane, and Envoy together. -| Consul Version | Consul Dataplane Version | Bundled Envoy Version | -| ------------------- | ------------------------ | ---------------------- | -| 1.14.x | 1.0.x | 1.24.x | +| Consul Version | Consul Dataplane Version (Bundled Envoy Version) | +| ------------------- | ------------------------------------------------- | +| 1.15.x | 1.1.x (Envoy 1.25.x), 1.0.x (Envoy 1.24.x) | +| 1.14.x | 1.0.x (Envoy 1.24.x) | ## Getting Started diff --git a/website/content/docs/k8s/compatibility.mdx b/website/content/docs/k8s/compatibility.mdx index 60f71bdcb853..343387156df0 100644 --- a/website/content/docs/k8s/compatibility.mdx +++ b/website/content/docs/k8s/compatibility.mdx @@ -15,9 +15,9 @@ Consul Kubernetes versions all of its components (`consul-k8s` CLI, `consul-k8s- | Consul Version | Compatible consul-k8s Versions | Compatible Kubernetes Versions | | -------------- | -------------------------------- | -------------------------------| +| 1.15.x | 1.1.x | 1.23.x - 1.26.x | | 1.14.x | 1.0.x | 1.22.x - 1.25.x | | 1.13.x | 0.49.x | 1.21.x - 1.24.x | -| 1.12.x | 0.43.0 - 0.49.x | 1.19.x - 1.22.x | ## Supported Envoy versions diff --git a/website/content/docs/k8s/helm.mdx b/website/content/docs/k8s/helm.mdx index b5eb83c0df02..602efba9e6de 100644 --- a/website/content/docs/k8s/helm.mdx +++ b/website/content/docs/k8s/helm.mdx @@ -58,7 +58,7 @@ Use these links to navigate to a particular top-level stanza. the prefix will be `-consul`. - `domain` ((#v-global-domain)) (`string: consul`) - The domain Consul will answer DNS queries for - (see `-domain` (https://www.consul.io/docs/agent/config/cli-flags#_domain)) and the domain services synced from + (Refer to [`-domain`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_domain)) and the domain services synced from Consul into Kubernetes will have, e.g. `service-name.service.consul`. - `peering` ((#v-global-peering)) - Configures the Cluster Peering feature. Requires Consul v1.14+ and Consul-K8s v1.0.0+. @@ -94,7 +94,7 @@ Use these links to navigate to a particular top-level stanza. - `imagePullSecrets` ((#v-global-imagepullsecrets)) (`array`) - Array of objects containing image pull secret names that will be applied to each service account. This can be used to reference image pull secrets if using a custom consul or consul-k8s-control-plane Docker image. - See https://kubernetes.io/docs/concepts/containers/images/#using-a-private-registry for reference. + Refer to https://kubernetes.io/docs/concepts/containers/images/#using-a-private-registry. Example: @@ -114,12 +114,13 @@ Use these links to navigate to a particular top-level stanza. https://github.com/hashicorp/consul/issues/1858. - `enablePodSecurityPolicies` ((#v-global-enablepodsecuritypolicies)) (`boolean: false`) - Controls whether pod security policies are created for the Consul components - created by this chart. See https://kubernetes.io/docs/concepts/policy/pod-security-policy/. + created by this chart. Refer to https://kubernetes.io/docs/concepts/policy/pod-security-policy/. - `secretsBackend` ((#v-global-secretsbackend)) - secretsBackend is used to configure Vault as the secrets backend for the Consul on Kubernetes installation. The Vault cluster needs to have the Kubernetes Auth Method, KV2 and PKI secrets engines enabled and have necessary secrets, policies and roles created prior to installing Consul. - See https://www.consul.io/docs/k8s/installation/vault for full instructions. + Refer to [Vault as the Secrets Backend](https://developer.hashicorp.com/consul/docs/k8s/deployment-configurations/vault) + documentation for full instructions. The Vault cluster _must_ not have the Consul cluster installed by this Helm chart as its storage backend as that would cause a circular dependency. @@ -177,11 +178,6 @@ Use these links to navigate to a particular top-level stanza. ``` and check the name of `metadata.name`. - - `controllerRole` ((#v-global-secretsbackend-vault-controllerrole)) (`string: ""`) - The Vault role to read Consul controller's webhook's - CA and issue a certificate and private key. - A Vault policy must be created which grants issue capabilities to - `global.secretsBackend.vault.controller.tlsCert.secretName`. - - `connectInjectRole` ((#v-global-secretsbackend-vault-connectinjectrole)) (`string: ""`) - The Vault role to read Consul connect-injector webhook's CA and issue a certificate and private key. A Vault policy must be created which grants issue capabilities to @@ -214,21 +210,21 @@ Use these links to navigate to a particular top-level stanza. The provider will be configured to use the Vault Kubernetes auth method and therefore requires the role provided by `global.secretsBackend.vault.consulServerRole` to have permissions to the root and intermediate PKI paths. - Please see https://www.consul.io/docs/connect/ca/vault#vault-acl-policies - for information on how to configure the Vault policies. + Please refer to [Vault ACL policies](https://developer.hashicorp.com/consul/docs/connect/ca/vault#vault-acl-policies) + documentation for information on how to configure the Vault policies. - `address` ((#v-global-secretsbackend-vault-connectca-address)) (`string: ""`) - The address of the Vault server. - `authMethodPath` ((#v-global-secretsbackend-vault-connectca-authmethodpath)) (`string: kubernetes`) - The mount path of the Kubernetes auth method in Vault. - `rootPKIPath` ((#v-global-secretsbackend-vault-connectca-rootpkipath)) (`string: ""`) - The path to a PKI secrets engine for the root certificate. - For more details, please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#rootpkipath). + For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#rootpkipath). - `intermediatePKIPath` ((#v-global-secretsbackend-vault-connectca-intermediatepkipath)) (`string: ""`) - The path to a PKI secrets engine for the generated intermediate certificate. - For more details, please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#intermediatepkipath). + For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#intermediatepkipath). - `additionalConfig` ((#v-global-secretsbackend-vault-connectca-additionalconfig)) (`string: {}`) - Additional Connect CA configuration in JSON format. - Please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#configuration) + Please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#configuration) for all configuration options available for that provider. Example: @@ -245,22 +241,6 @@ Use these links to navigate to a particular top-level stanza. } ``` - - `controller` ((#v-global-secretsbackend-vault-controller)) - - - `tlsCert` ((#v-global-secretsbackend-vault-controller-tlscert)) - Configuration to the Vault Secret that Kubernetes will use on - Kubernetes CRD creation, deletion, and update, to get TLS certificates - used issued from vault to send webhooks to the controller. - - - `secretName` ((#v-global-secretsbackend-vault-controller-tlscert-secretname)) (`string: null`) - The Vault secret path that issues TLS certificates for controller - webhooks. - - - `caCert` ((#v-global-secretsbackend-vault-controller-cacert)) - Configuration to the Vault Secret that Kubernetes will use on - Kubernetes CRD creation, deletion, and update, to get CA certificates - used issued from vault to send webhooks to the controller. - - - `secretName` ((#v-global-secretsbackend-vault-controller-cacert-secretname)) (`string: null`) - The Vault secret path that contains the CA certificate for controller - webhooks. - - `connectInject` ((#v-global-secretsbackend-vault-connectinject)) - `caCert` ((#v-global-secretsbackend-vault-connectinject-cacert)) - Configuration to the Vault Secret that Kubernetes uses on @@ -278,7 +258,7 @@ Use these links to navigate to a particular top-level stanza. inject webhooks. - `gossipEncryption` ((#v-global-gossipencryption)) - Configures Consul's gossip encryption key. - (see `-encrypt` (https://www.consul.io/docs/agent/config/cli-flags#_encrypt)). + (Refer to [`-encrypt`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_encrypt)). By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually. The recommended method is to automatically generate the key. To automatically generate and set a gossip encryption key, set autoGenerate to true. @@ -286,7 +266,7 @@ Use these links to navigate to a particular top-level stanza. To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate a key, saving this as a Kubernetes secret or Vault secret path and key. If `global.secretsBackend.vault.enabled=true`, be sure to add the "data" component of the secretName path as required by - the Vault KV-2 secrets engine [see example]. + the Vault KV-2 secrets engine [refer to example]. ```shell-session $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen) @@ -309,10 +289,10 @@ Use these links to navigate to a particular top-level stanza. - `recursors` ((#v-global-recursors)) (`array: []`) - A list of addresses of upstream DNS servers that are used to recursively resolve DNS queries. These values are given as `-recursor` flags to Consul servers and clients. - See https://www.consul.io/docs/agent/config/cli-flags#_recursor for more details. + Refer to [`-recursor`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_recursor) for more details. If this is an empty array (the default), then Consul DNS will only resolve queries for the Consul top level domain (by default `.consul`). - - `tls` ((#v-global-tls)) - Enables TLS (https://learn.hashicorp.com/tutorials/consul/tls-encryption-secure) + - `tls` ((#v-global-tls)) - Enables [TLS](https://developer.hashicorp.com/consul/tutorials/security/tls-encryption-secure) across the cluster to verify authenticity of the Consul servers and clients. Requires Consul v1.4.1+. @@ -336,7 +316,7 @@ Use these links to navigate to a particular top-level stanza. - `verify` ((#v-global-tls-verify)) (`boolean: true`) - If true, `verify_outgoing`, `verify_server_hostname`, and `verify_incoming` for internal RPC communication will be set to `true` for Consul servers and clients. Set this to false to incrementally roll out TLS on an existing Consul cluster. - Please see https://consul.io/docs/k8s/operations/tls-on-existing-cluster + Please refer to [TLS on existing clusters](https://developer.hashicorp.com/consul/docs/k8s/operations/tls-on-existing-cluster) for more details. - `httpsOnly` ((#v-global-tls-httpsonly)) (`boolean: true`) - If true, the Helm chart will configure Consul to disable the HTTP port on @@ -372,8 +352,9 @@ Use these links to navigate to a particular top-level stanza. Note that we need the CA key so that we can generate server and client certificates. It is particularly important for the client certificates since they need to have host IPs - as Subject Alternative Names. In the future, we may support bringing your own server - certificates. + as Subject Alternative Names. If you are setting server certs yourself via `server.serverCert` + and you are not enabling clients (or clients are enabled with autoEncrypt) then you do not + need to provide the CA key. - `secretName` ((#v-global-tls-cakey-secretname)) (`string: null`) - The name of the Kubernetes or Vault secret that holds the CA key. @@ -430,9 +411,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-global-acls-tolerations)) (`string: ""`) - tolerations configures the taints and tolerations for the server-acl-init and server-acl-init-cleanup jobs. This should be a multi-line string matching the - Tolerations (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - - `nodeSelector` ((#v-global-acls-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-global-acls-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for the server-acl-init and server-acl-init-cleanup jobs pod assignment, formatted as a multi-line string. Example: @@ -482,7 +463,7 @@ Use these links to navigate to a particular top-level stanza. This address must be reachable from the Consul servers in the primary datacenter. This auth method will be used to provision ACL tokens for Consul components and is different from the one used by the Consul Service Mesh. - Please see the [Kubernetes Auth Method documentation](https://consul.io/docs/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). You can retrieve this value from your `kubeconfig` by running: @@ -593,7 +574,7 @@ Use these links to navigate to a particular top-level stanza. Consul server agents. - `replicas` ((#v-server-replicas)) (`integer: 1`) - The number of server agents to run. This determines the fault tolerance of - the cluster. Please see the deployment table (https://consul.io/docs/internals/consensus#deployment-table) + the cluster. Please refer to the [deployment table](https://developer.hashicorp.com/consul/docs/architecture/consensus#deployment-table) for more information. - `bootstrapExpect` ((#v-server-bootstrapexpect)) (`int: null`) - The number of servers that are expected to be running. @@ -632,8 +613,8 @@ Use these links to navigate to a particular top-level stanza. Vault Secrets backend: If you are using Vault as a secrets backend, a Vault Policy must be created which allows `["create", "update"]` capabilities on the PKI issuing endpoint, which is usually of the form `pki/issue/consul-server`. - Please see the following guide for steps to generate a compatible certificate: - https://learn.hashicorp.com/tutorials/consul/vault-pki-consul-secure-tls + Complete [this tutorial](https://developer.hashicorp.com/consul/tutorials/vault-secure/vault-pki-consul-secure-tls) + to learn how to generate a compatible certificate. Note: when using TLS, both the `server.serverCert` and `global.tls.caCert` which points to the CA endpoint of this PKI engine must be provided. @@ -672,15 +653,15 @@ Use these links to navigate to a particular top-level stanza. storage classes, the PersistentVolumeClaims would need to be manually created. A `null` value will use the Kubernetes cluster's default StorageClass. If a default StorageClass does not exist, you will need to create one. - Refer to the [Read/Write Tuning](https://www.consul.io/docs/install/performance#read-write-tuning) + Refer to the [Read/Write Tuning](https://developer.hashicorp.com/consul/docs/install/performance#read-write-tuning) section of the Server Performance Requirements documentation for considerations around choosing a performant storage class. - ~> **Note:** The [Reference Architecture](https://learn.hashicorp.com/tutorials/consul/reference-architecture#hardware-sizing-for-consul-servers) + ~> **Note:** The [Reference Architecture](https://developer.hashicorp.com/consul/tutorials/production-deploy/reference-architecture#hardware-sizing-for-consul-servers) contains best practices and recommendations for selecting suitable hardware sizes for your Consul servers. - - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable Connect (https://consul.io/docs/connect). Setting this to true + - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable [Connect](https://developer.hashicorp.com/consul/docs/connect). Setting this to true _will not_ automatically secure pod communication, this setting will only enable usage of the feature. Consul will automatically initialize a new CA and set of certificates. Additional Connect settings can be configured @@ -699,7 +680,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-server-resources)) (`map`) - The resource requests (CPU, memory, etc.) for each of the server agents. This should be a YAML map corresponding to a Kubernetes - ResourceRequirements (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core) + [`ResourceRequirements``](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core) object. NOTE: The use of a YAML string is deprecated. Example: @@ -730,11 +711,12 @@ Use these links to navigate to a particular top-level stanza. - `updatePartition` ((#v-server-updatepartition)) (`integer: 0`) - This value is used to carefully control a rolling update of Consul server agents. This value specifies the - partition (https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) - for performing a rolling update. Please read the linked Kubernetes documentation - and https://www.consul.io/docs/k8s/upgrade#upgrading-consul-servers for more information. + [partition](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) + for performing a rolling update. Please read the linked Kubernetes + and [Upgrade Consul](https://developer.hashicorp.com/consul/docs/k8s/upgrade#upgrading-consul-servers) + documentation for more information. - - `disruptionBudget` ((#v-server-disruptionbudget)) - This configures the PodDisruptionBudget (https://kubernetes.io/docs/tasks/run-application/configure-pdb/) + - `disruptionBudget` ((#v-server-disruptionbudget)) - This configures the [`PodDisruptionBudget`](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) for the server cluster. - `enabled` ((#v-server-disruptionbudget-enabled)) (`boolean: true`) - Enables registering a PodDisruptionBudget for the server @@ -747,7 +729,7 @@ Use these links to navigate to a particular top-level stanza. --set 'server.disruptionBudget.maxUnavailable=0'` flag to the helm chart installation command because of a limitation in the Helm templating language. - - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra JSON configuration (https://consul.io/docs/agent/options) for Consul + - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul servers. This will be saved as-is into a ConfigMap that is read by the Consul server agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -803,7 +785,7 @@ Use these links to navigate to a particular top-level stanza. - ... ``` - - `affinity` ((#v-server-affinity)) (`string`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-server-affinity)) (`string`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for server pods. It defaults to allowing only a single server pod on each node, which minimizes risk of the cluster becoming unusable if a node is lost. If you need to run more pods per node (for example, testing on Minikube), set this value @@ -824,12 +806,14 @@ Use these links to navigate to a particular top-level stanza. ``` - `tolerations` ((#v-server-tolerations)) (`string: ""`) - Toleration settings for server pods. This - should be a multi-line string matching the Tolerations - (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + should be a multi-line string matching the + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + array in a Pod spec. - `topologySpreadConstraints` ((#v-server-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for server pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -847,7 +831,7 @@ Use these links to navigate to a particular top-level stanza. component: server ``` - - `nodeSelector` ((#v-server-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-server-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for server pod assignment, formatted as a multi-line string. Example: @@ -858,7 +842,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-server-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to server pods. - `extraLabels` ((#v-server-extralabels)) (`map`) - Extra labels to attach to the server pods. This should be a YAML map. @@ -921,19 +905,19 @@ Use these links to navigate to a particular top-level stanza. feature, in case kubernetes cluster is behind egress http proxies. Additionally, it could be used to configure custom consul parameters. - - `snapshotAgent` ((#v-server-snapshotagent)) - Values for setting up and running snapshot agents - (https://consul.io/commands/snapshot/agent) + - `snapshotAgent` ((#v-server-snapshotagent)) - Values for setting up and running + [snapshot agents](https://developer.hashicorp.com/consul/commands/snapshot/agent) within the Consul clusters. They run as a sidecar with Consul servers. - `enabled` ((#v-server-snapshotagent-enabled)) (`boolean: false`) - If true, the chart will install resources necessary to run the snapshot agent. - `interval` ((#v-server-snapshotagent-interval)) (`string: 1h`) - Interval at which to perform snapshots. - See https://www.consul.io/commands/snapshot/agent#interval + Refer to [`interval`](https://developer.hashicorp.com/consul/commands/snapshot/agent#interval) - `configSecret` ((#v-server-snapshotagent-configsecret)) - A Kubernetes or Vault secret that should be manually created to contain the entire config to be used on the snapshot agent. This is the preferred method of configuration since there are usually storage - credentials present. Please see Snapshot agent config (https://consul.io/commands/snapshot/agent#config-file-options) + credentials present. Please refer to the [Snapshot agent config](https://developer.hashicorp.com/consul/commands/snapshot/agent#config-file-options) for details. - `secretName` ((#v-server-snapshotagent-configsecret-secretname)) (`string: null`) - The name of the Kubernetes secret or Vault secret path that holds the snapshot agent config. @@ -991,7 +975,7 @@ Use these links to navigate to a particular top-level stanza. - `k8sAuthMethodHost` ((#v-externalservers-k8sauthmethodhost)) (`string: null`) - If you are setting `global.acls.manageSystemACLs` and `connectInject.enabled` to true, set `k8sAuthMethodHost` to the address of the Kubernetes API server. This address must be reachable from the Consul servers. - Please see the Kubernetes Auth Method documentation (https://consul.io/docs/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). You could retrieve this value from your `kubeconfig` by running: @@ -1014,7 +998,7 @@ Use these links to navigate to a particular top-level stanza. - `image` ((#v-client-image)) (`string: null`) - The name of the Docker image (including any tag) for the containers running Consul client agents. - - `join` ((#v-client-join)) (`array: null`) - A list of valid `-retry-join` values (https://www.consul.io/docs/agent/config/cli-flags#_retry_join). + - `join` ((#v-client-join)) (`array: null`) - A list of valid [`-retry-join` values](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_retry_join). If this is `null` (default), then the clients will attempt to automatically join the server cluster running within Kubernetes. This means that with `server.enabled` set to true, clients will automatically @@ -1035,7 +1019,7 @@ Use these links to navigate to a particular top-level stanza. required for Connect. - `nodeMeta` ((#v-client-nodemeta)) - nodeMeta specifies an arbitrary metadata key/value pair to associate with the node - (see https://www.consul.io/docs/agent/config/cli-flags#_node_meta) + (refer to [`-node-meta`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_node_meta)) - `pod-name` ((#v-client-nodemeta-pod-name)) (`string: ${HOSTNAME}`) @@ -1079,7 +1063,7 @@ Use these links to navigate to a particular top-level stanza. - `tlsInit` ((#v-client-containersecuritycontext-tlsinit)) (`map`) - The tls-init initContainer - - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra JSON configuration (https://consul.io/docs/agent/options) for Consul + - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul clients. This will be saved as-is into a ConfigMap that is read by the Consul client agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -1172,7 +1156,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-client-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to client pods. - `annotations` ((#v-client-annotations)) (`string: null`) - This value defines additional annotations for @@ -1199,7 +1183,7 @@ Use these links to navigate to a particular top-level stanza. feature, in case kubernetes cluster is behind egress http proxies. Additionally, it could be used to configure custom consul parameters. - - `dnsPolicy` ((#v-client-dnspolicy)) (`string: null`) - This value defines the Pod DNS policy (https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy) + - `dnsPolicy` ((#v-client-dnspolicy)) (`string: null`) - This value defines the [Pod DNS policy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy) for client pods to use. - `hostNetwork` ((#v-client-hostnetwork)) (`boolean: false`) - hostNetwork defines whether or not we use host networking instead of hostPort in the event @@ -1209,7 +1193,8 @@ Use these links to navigate to a particular top-level stanza. combined with `dnsPolicy: ClusterFirstWithHostNet` - `updateStrategy` ((#v-client-updatestrategy)) (`string: null`) - updateStrategy for the DaemonSet. - See https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy. + Refer to the Kubernetes [Daemonset upgrade strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) + documentation. This should be a multi-line string mapping directly to the updateStrategy Example: @@ -1307,7 +1292,7 @@ Use these links to navigate to a particular top-level stanza. - `ingressClassName` ((#v-ui-ingress-ingressclassname)) (`string: ""`) - Optionally set the ingressClassName. - - `pathType` ((#v-ui-ingress-pathtype)) (`string: Prefix`) - pathType override - see: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types + - `pathType` ((#v-ui-ingress-pathtype)) (`string: Prefix`) - pathType override - refer to: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types - `hosts` ((#v-ui-ingress-hosts)) (`array`) - hosts is a list of host name to create Ingress rules. @@ -1343,16 +1328,17 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-ui-metrics-enabled)) (`boolean: global.metrics.enabled`) - Enable displaying metrics in the UI. The default value of "-" will inherit from `global.metrics.enabled` value. - - `provider` ((#v-ui-metrics-provider)) (`string: prometheus`) - Provider for metrics. See - https://www.consul.io/docs/agent/options#ui_config_metrics_provider + - `provider` ((#v-ui-metrics-provider)) (`string: prometheus`) - Provider for metrics. Refer to + [`metrics_provider`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_metrics_provider) This value is only used if `ui.enabled` is set to true. - `baseURL` ((#v-ui-metrics-baseurl)) (`string: http://prometheus-server`) - baseURL is the URL of the prometheus server, usually the service URL. This value is only used if `ui.enabled` is set to true. - - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to https://www.consul.io/docs/agent/options#ui_config_dashboard_url_templates configuration. + - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to [`dashboard_url_templates`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates) + configuration. - - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets https://www.consul.io/docs/agent/options#ui_config_dashboard_url_templates_service. + - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets [`dashboardURLTemplates.service`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates_service). ### syncCatalog ((#h-synccatalog)) @@ -1372,8 +1358,8 @@ Use these links to navigate to a particular top-level stanza. to run the sync program. - `default` ((#v-synccatalog-default)) (`boolean: true`) - If true, all valid services in K8S are - synced by default. If false, the service must be annotated - (https://consul.io/docs/k8s/service-sync#sync-enable-disable) properly to sync. + synced by default. If false, the service must be [annotated](https://developer.hashicorp.com/consul/docs/k8s/service-sync#enable-and-disable-sync) + properly to sync. In either case an annotation can override the default. - `priorityClassName` ((#v-synccatalog-priorityclassname)) (`string: ""`) - Optional priorityClassName. @@ -1486,7 +1472,7 @@ Use these links to navigate to a particular top-level stanza. - `secretKey` ((#v-synccatalog-aclsynctoken-secretkey)) (`string: null`) - The key within the Kubernetes secret that holds the acl sync token. - - `nodeSelector` ((#v-synccatalog-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-synccatalog-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for catalog sync pod assignment, formatted as a multi-line string. Example: @@ -1552,7 +1538,7 @@ Use these links to navigate to a particular top-level stanza. - `default` ((#v-connectinject-default)) (`boolean: false`) - If true, the injector will inject the Connect sidecar into all pods by default. Otherwise, pods must specify the - injection annotation (https://consul.io/docs/k8s/connect#consul-hashicorp-com-connect-inject) + [injection annotation](https://developer.hashicorp.com/consul/docs/k8s/connect#consul-hashicorp-com-connect-inject) to opt-in to Connect injection. If this is true, pods can use the same annotation to explicitly opt-out of injection. @@ -1570,7 +1556,7 @@ Use these links to navigate to a particular top-level stanza. This value is also overridable via the "consul.hashicorp.com/transparent-proxy-overwrite-probes" annotation. Note: This value has no effect if transparent proxy is disabled on the pod. - - `disruptionBudget` ((#v-connectinject-disruptionbudget)) - This configures the PodDisruptionBudget (https://kubernetes.io/docs/tasks/run-application/configure-pdb/) + - `disruptionBudget` ((#v-connectinject-disruptionbudget)) - This configures the [`PodDisruptionBudget`](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) for the service mesh sidecar injector. - `enabled` ((#v-connectinject-disruptionbudget-enabled)) (`boolean: true`) - This will enable/disable registering a PodDisruptionBudget for the @@ -1629,7 +1615,8 @@ Use these links to navigate to a particular top-level stanza. by the OpenShift platform. - `updateStrategy` ((#v-connectinject-cni-updatestrategy)) (`string: null`) - updateStrategy for the CNI installer DaemonSet. - See https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy. + Refer to the Kubernetes [Daemonset upgrade strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) + documentation. This should be a multi-line string mapping directly to the updateStrategy Example: @@ -1742,12 +1729,12 @@ Use these links to navigate to a particular top-level stanza. - `namespaceSelector` ((#v-connectinject-namespaceselector)) (`string`) - Selector for restricting the webhook to only specific namespaces. Use with `connectInject.default: true` to automatically inject all pods in namespaces that match the selector. This should be set to a multiline string. - See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + Refer to https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector for more details. - By default, we exclude the kube-system namespace since usually users won't - want those pods injected and also the local-path-storage namespace so that - Kind (Kubernetes In Docker) can provision Pods used to create PVCs. + By default, we exclude kube-system since usually users won't + want those pods injected and local-path-storage and openebs so that + Kind (Kubernetes In Docker) and [OpenEBS](https://openebs.io/) respectively can provision Pods used to create PVCs. Note that this exclusion is only supported in Kubernetes v1.21.1+. Example: @@ -1829,8 +1816,8 @@ Use these links to navigate to a particular top-level stanza. If set to an empty string all service accounts can log in. This only has effect if ACLs are enabled. - See https://www.consul.io/docs/acl/acl-auth-methods.html#binding-rules - and https://www.consul.io/docs/acl/auth-methods/kubernetes.html#trusted-identity-attributes + Refer to Auth methods [Binding rules](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods#binding-rules) + and [Trusted identiy attributes](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes#trusted-identity-attributes) for more details. Requires Consul >= v1.5. @@ -1856,7 +1843,7 @@ Use these links to navigate to a particular top-level stanza. leads to unnecessary thread and memory usage and leaves unnecessary idle connections open. It is advised to keep this number low for sidecars and high for edge proxies. This will control the `--concurrency` flag to Envoy. - For additional information see also: https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310 + For additional information, refer to https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310 This setting can be overridden on a per-pod basis via this annotation: - `consul.hashicorp.com/consul-envoy-proxy-concurrency` @@ -1924,7 +1911,7 @@ Use these links to navigate to a particular top-level stanza. - `port` ((#v-meshgateway-wanaddress-port)) (`integer: 443`) - Port that gets registered for WAN traffic. If source is set to "Service" then this setting will have no effect. - See the documentation for source as to which port will be used in that + Refer to the documentation for source as to which port will be used in that case. - `static` ((#v-meshgateway-wanaddress-static)) (`string: ""`) - If source is set to "Static" then this value will be used as the WAN @@ -1989,7 +1976,7 @@ Use these links to navigate to a particular top-level stanza. - `initServiceInitContainer` ((#v-meshgateway-initserviceinitcontainer)) (`map`) - The resource settings for the `service-init` init container. - - `affinity` ((#v-meshgateway-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-meshgateway-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for mesh gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2011,8 +1998,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-meshgateway-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-meshgateway-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for mesh gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2102,7 +2090,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-ingressgateways-defaults-resources)) (`map`) - Resource limits for all ingress gateway pods - - `affinity` ((#v-ingressgateways-defaults-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-ingressgateways-defaults-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for ingress gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2124,8 +2112,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-ingressgateways-defaults-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-ingressgateways-defaults-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for ingress gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2208,7 +2197,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-terminatinggateways-defaults-resources)) (`map`) - Resource limits for all terminating gateway pods - - `affinity` ((#v-terminatinggateways-defaults-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-terminatinggateways-defaults-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for terminating gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2230,8 +2219,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-terminatinggateways-defaults-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-terminatinggateways-defaults-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for terminating gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2306,7 +2296,7 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-apigateway-managedgatewayclass-enabled)) (`boolean: true`) - When true a GatewayClass is configured to automatically work with Consul as installed by helm. - - `nodeSelector` ((#v-apigateway-managedgatewayclass-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-apigateway-managedgatewayclass-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for gateway pod assignment, formatted as a multi-line string. Example: @@ -2370,10 +2360,10 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-apigateway-controller-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to api-gateway-controller pods. - - `nodeSelector` ((#v-apigateway-controller-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-apigateway-controller-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for api-gateway-controller pod assignment, formatted as a multi-line string. Example: @@ -2384,7 +2374,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `tolerations` ((#v-apigateway-controller-tolerations)) (`string: null`) - This value defines the tolerations for api-gateway-controller pod, this should be a multi-line string matching the - Tolerations (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - `service` ((#v-apigateway-controller-service)) - Configuration for the Service created for the api-gateway-controller @@ -2408,7 +2398,7 @@ Use these links to navigate to a particular top-level stanza. This should be a multi-line string matching the Toleration array in a PodSpec. - - `nodeSelector` ((#v-webhookcertmanager-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-webhookcertmanager-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for the webhook-cert-manager pod assignment, formatted as a multi-line string. Example: From 31e094b36e3e369834d3b3562220302afc7f36f6 Mon Sep 17 00:00:00 2001 From: cskh Date: Tue, 28 Feb 2023 11:31:52 -0500 Subject: [PATCH 057/421] upgrade test: consolidate resolver test cases (#16443) (#16457) --- .../consul-container/libs/assert/envoy.go | 33 +- .../resolver_default_subset_test.go | 465 +++++++++++++----- .../resolver_subset_onlypassing_test.go | 196 -------- .../resolver_subset_redirect_test.go | 230 --------- .../consul-container/test/upgrade/upgrade.go | 3 + 5 files changed, 382 insertions(+), 545 deletions(-) delete mode 100644 test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go delete mode 100644 test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go create mode 100644 test/integration/consul-container/test/upgrade/upgrade.go diff --git a/test/integration/consul-container/libs/assert/envoy.go b/test/integration/consul-container/libs/assert/envoy.go index 72c0ba990fbf..1fdf61feae3c 100644 --- a/test/integration/consul-container/libs/assert/envoy.go +++ b/test/integration/consul-container/libs/assert/envoy.go @@ -218,6 +218,23 @@ func AssertEnvoyPresentsCertURI(t *testing.T, port int, serviceName string) { } } +// AssertEnvoyRunning assert the envoy is running by querying its stats page +func AssertEnvoyRunning(t *testing.T, port int) { + var ( + err error + ) + failer := func() *retry.Timer { + return &retry.Timer{Timeout: 10 * time.Second, Wait: 500 * time.Millisecond} + } + + retry.RunWith(failer(), t, func(r *retry.R) { + _, _, err = GetEnvoyOutput(port, "stats", nil) + if err != nil { + r.Fatal("could not fetch envoy stats") + } + }) +} + func GetEnvoyOutput(port int, path string, query map[string]string) (string, int, error) { client := cleanhttp.DefaultClient() var u url.URL @@ -260,10 +277,16 @@ func sanitizeResult(s string) []string { // AssertServiceHasHealthyInstances asserts the number of instances of service equals count for a given service. // https://developer.hashicorp.com/consul/docs/connect/config-entries/service-resolver#onlypassing func AssertServiceHasHealthyInstances(t *testing.T, node libcluster.Agent, service string, onlypassing bool, count int) { - services, _, err := node.GetClient().Health().Service(service, "", onlypassing, nil) - require.NoError(t, err) - for _, v := range services { - fmt.Printf("%s service status: %s\n", v.Service.ID, v.Checks.AggregatedStatus()) + failer := func() *retry.Timer { + return &retry.Timer{Timeout: 10 * time.Second, Wait: 500 * time.Millisecond} } - require.Equal(t, count, len(services)) + + retry.RunWith(failer(), t, func(r *retry.R) { + services, _, err := node.GetClient().Health().Service(service, "", onlypassing, nil) + require.NoError(r, err) + for _, v := range services { + fmt.Printf("%s service status: %s\n", v.Service.ID, v.Checks.AggregatedStatus()) + } + require.Equal(r, count, len(services)) + }) } diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go index 182c1cc12783..2203954a44d3 100644 --- a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go @@ -3,8 +3,8 @@ package upgrade import ( "context" "fmt" - "net/http" "testing" + "time" "github.com/hashicorp/consul/api" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" @@ -12,137 +12,394 @@ import ( libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" libutils "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + upgrade "github.com/hashicorp/consul/test/integration/consul-container/test/upgrade" "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" - "gotest.tools/assert" ) -// TestTrafficManagement_ServiceResolverDefaultSubset Summary -// This test starts up 3 servers and 1 client in the same datacenter. +// TestTrafficManagement_ServiceResolver tests that upgraded cluster inherits and interpret +// the resolver config entry correctly. // -// Steps: -// - Create a single agent cluster. -// - Create one static-server and 2 subsets and 1 client and sidecar, then register them with Consul -// - Validate static-server and 2 subsets are and proxy admin endpoint is healthy - 3 instances -// - Validate static servers proxy listeners should be up and have right certs -func TestTrafficManagement_ServiceResolverDefaultSubset(t *testing.T) { +// The basic topology is a cluster with one static-client and one static-server. Addtional +// services and resolver can be added to the create func() for each test cases. +func TestTrafficManagement_ServiceResolver(t *testing.T) { t.Parallel() - var responseFormat = map[string]string{"format": "json"} - type testcase struct { - oldversion string - targetVersion string + name string + // create creates addtional resources in the cluster depending on cases, e.g., static-client, + // static server, and config-entries. It returns the proxy services of the client, an assertation + // function to be called to verify the resources, and a restartFn to be called after upgrade. + create func(*libcluster.Cluster, libservice.Service) (libservice.Service, func(), func(), error) + // extraAssertion adds additional assertion function to the common resources across cases. + // common resources includes static-client in dialing cluster, and static-server in accepting cluster. + // + // extraAssertion needs to be run before and after upgrade + extraAssertion func(libservice.Service) } tcs := []testcase{ { - oldversion: "1.13", - targetVersion: libutils.TargetVersion, + // Test resolver directs traffic to default subset + // - Create 2 additional static-server instances: one in V1 subset and the other in V2 subset + // - resolver directs traffic to the default subset, which is V2. + name: "resolver default subset", + create: func(cluster *libcluster.Cluster, clientConnectProxy libservice.Service) (libservice.Service, func(), func(), error) { + node := cluster.Agents[0] + client := node.GetClient() + + // Create static-server-v1 and static-server-v2 + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8081, + GRPCPort: 8078, + } + _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) + require.NoError(t, err) + + serviceOptsV2 := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server-v2", + Meta: map[string]string{"version": "v2"}, + HTTPPort: 8082, + GRPCPort: 8077, + } + _, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "static-server") + + // TODO: verify the number of instance of static-server is 3 + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 3) + + // Register service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + DefaultSubset: "v2", + Subsets: map[string]api.ServiceResolverSubset{ + "v1": { + Filter: "Service.Meta.version == v1", + }, + "v2": { + Filter: "Service.Meta.version == v2", + }, + }, + } + err = cluster.ConfigEntryWrite(serviceResolver) + if err != nil { + return nil, nil, nil, fmt.Errorf("error writing config entry %s", err) + } + + _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() + _, serverAdminPortV2 := serverConnectProxyV2.GetAdminAddr() + + restartFn := func() { + require.NoError(t, serverConnectProxyV1.Restart()) + require.NoError(t, serverConnectProxyV2.Restart()) + } + + _, adminPort := clientConnectProxy.GetAdminAddr() + assertionFn := func() { + libassert.AssertEnvoyRunning(t, serverAdminPortV1) + libassert.AssertEnvoyRunning(t, serverAdminPortV2) + + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server") + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, "static-server") + + libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) + + // assert static-server proxies should be healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 3) + } + return nil, assertionFn, restartFn, nil + }, + extraAssertion: func(clientConnectProxy libservice.Service) { + _, port := clientConnectProxy.GetAddr() + _, adminPort := clientConnectProxy.GetAdminAddr() + + libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) + + // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2") + }, + }, + { + // Test resolver resolves service instance based on their check status + // - Create one addtional static-server with checks and V1 subset + // - resolver directs traffic to "test" service + name: "resolver default onlypassing", + create: func(cluster *libcluster.Cluster, clientConnectProxy libservice.Service) (libservice.Service, func(), func(), error) { + node := cluster.Agents[0] + + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8081, + GRPCPort: 8078, + Checks: libservice.Checks{ + Name: "main", + TTL: "30m", + }, + Connect: libservice.SidecarService{ + Port: 21011, + }, + } + _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecarWithChecks(node, serviceOptsV1) + require.NoError(t, err) + + // Register service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + DefaultSubset: "test", + Subsets: map[string]api.ServiceResolverSubset{ + "test": { + OnlyPassing: true, + }, + }, + ConnectTimeout: 120 * time.Second, + } + _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() + + restartFn := func() { + require.NoError(t, serverConnectProxyV1.Restart()) + } + + _, port := clientConnectProxy.GetAddr() + _, adminPort := clientConnectProxy.GetAdminAddr() + assertionFn := func() { + // force static-server-v1 into a warning state + err = node.GetClient().Agent().UpdateTTL("service:static-server-v1", "", "warn") + require.NoError(t, err) + + // ########################### + // ## with onlypassing=true + // assert only one static-server proxy is healthy + err = cluster.ConfigEntryWrite(serviceResolver) + require.NoError(t, err) + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1) + + libassert.AssertEnvoyRunning(t, serverAdminPortV1) + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server") + + // assert static-server proxies should be healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1) + + // static-client upstream should have 1 healthy endpoint for test.static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "HEALTHY", 1) + + // static-client upstream should have 1 unhealthy endpoint for test.static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "UNHEALTHY", 1) + + // static-client upstream should connect to static-server since it is passing + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName) + + // ########################### + // ## with onlypassing=false + // revert to OnlyPassing=false by deleting the config + err = cluster.ConfigEntryDelete(serviceResolver) + require.NoError(t, err) + + // Consul health check assert only one static-server proxy is healthy when onlyPassing is false + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, false, 2) + + // Although the service status is in warning state, when onlypassing is set to false Envoy + // health check returns all service instances with "warning" or "passing" state as Healthy enpoints + libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "HEALTHY", 2) + + // static-client upstream should have 0 unhealthy endpoint for static-server + libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "UNHEALTHY", 0) + + } + return nil, assertionFn, restartFn, nil + }, + extraAssertion: func(clientConnectProxy libservice.Service) { + }, }, { - oldversion: "1.14", - targetVersion: libutils.TargetVersion, + // Test resolver directs traffic to default subset + // - Create 3 static-server-2 server instances: one in V1, one in V2, one without any version + // - service2Resolver directs traffic to static-server-2-v2 + name: "resolver subset redirect", + create: func(cluster *libcluster.Cluster, clientConnectProxy libservice.Service) (libservice.Service, func(), func(), error) { + node := cluster.Agents[0] + client := node.GetClient() + + serviceOpts2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2", + HTTPPort: 8081, + GRPCPort: 8078, + } + _, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8082, + GRPCPort: 8077, + } + _, server2ConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) + require.NoError(t, err) + + serviceOptsV2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v2", + Meta: map[string]string{"version": "v2"}, + HTTPPort: 8083, + GRPCPort: 8076, + } + _, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + // Register static-server service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServer2ServiceName, + Subsets: map[string]api.ServiceResolverSubset{ + "v1": { + Filter: "Service.Meta.version == v1", + }, + "v2": { + Filter: "Service.Meta.version == v2", + }, + }, + } + err = cluster.ConfigEntryWrite(serviceResolver) + require.NoError(t, err) + + // Register static-server-2 service resolver to redirect traffic + // from static-server to static-server-2-v2 + service2Resolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServer2ServiceName, + ServiceSubset: "v2", + }, + } + err = cluster.ConfigEntryWrite(service2Resolver) + require.NoError(t, err) + + _, server2AdminPort := server2ConnectProxy.GetAdminAddr() + _, server2AdminPortV1 := server2ConnectProxyV1.GetAdminAddr() + _, server2AdminPortV2 := server2ConnectProxyV2.GetAdminAddr() + + restartFn := func() { + require.NoErrorf(t, server2ConnectProxy.Restart(), "%s", server2ConnectProxy.GetName()) + require.NoErrorf(t, server2ConnectProxyV1.Restart(), "%s", server2ConnectProxyV1.GetName()) + require.NoErrorf(t, server2ConnectProxyV2.Restart(), "%s", server2ConnectProxyV2.GetName()) + } + + assertionFn := func() { + // assert 3 static-server-2 instances are healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServer2ServiceName, false, 3) + + libassert.AssertEnvoyRunning(t, server2AdminPort) + libassert.AssertEnvoyRunning(t, server2AdminPortV1) + libassert.AssertEnvoyRunning(t, server2AdminPortV2) + + libassert.AssertEnvoyPresentsCertURI(t, server2AdminPort, libservice.StaticServer2ServiceName) + libassert.AssertEnvoyPresentsCertURI(t, server2AdminPortV1, libservice.StaticServer2ServiceName) + libassert.AssertEnvoyPresentsCertURI(t, server2AdminPortV2, libservice.StaticServer2ServiceName) + + // assert static-server proxies should be healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServer2ServiceName, true, 3) + } + return nil, assertionFn, restartFn, nil + }, + extraAssertion: func(clientConnectProxy libservice.Service) { + _, appPort := clientConnectProxy.GetAddr() + _, adminPort := clientConnectProxy.GetAdminAddr() + + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPort), "static-server-2-v2") + libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server-2.default", "HEALTHY", 1) + }, }, } - run := func(t *testing.T, tc testcase) { + run := func(t *testing.T, tc testcase, oldVersion, targetVersion string) { buildOpts := &libcluster.BuildOptions{ - ConsulVersion: tc.oldversion, + ConsulVersion: oldVersion, Datacenter: "dc1", InjectAutoEncryption: true, } // If version < 1.14 disable AutoEncryption - oldVersion, _ := version.NewVersion(tc.oldversion) - if oldVersion.LessThan(libutils.Version_1_14) { + oldVersionTmp, _ := version.NewVersion(oldVersion) + if oldVersionTmp.LessThan(libutils.Version_1_14) { buildOpts.InjectAutoEncryption = false } cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) + node := cluster.Agents[0] + client := node.GetClient() - // Register service resolver - serviceResolver := &api.ServiceResolverConfigEntry{ - Kind: api.ServiceResolver, - Name: libservice.StaticServerServiceName, - DefaultSubset: "v2", - Subsets: map[string]api.ServiceResolverSubset{ - "v1": { - Filter: "Service.Meta.version == v1", - }, - "v2": { - Filter: "Service.Meta.version == v2", - }, - }, - } - err := cluster.ConfigEntryWrite(serviceResolver) + staticClientProxy, staticServerProxy, err := createStaticClientAndServer(cluster) require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) - serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy := createService(t, cluster) + err = cluster.ConfigEntryWrite(&api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: "global", + Config: map[string]interface{}{ + "protocol": "http", + }, + }) + require.NoError(t, err) - _, port := clientConnectProxy.GetAddr() - _, adminPort := clientConnectProxy.GetAdminAddr() - _, serverAdminPort := serverConnectProxy.GetAdminAddr() - _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() - _, serverAdminPortV2 := serverConnectProxyV2.GetAdminAddr() + _, port := staticClientProxy.GetAddr() + _, adminPort := staticClientProxy.GetAdminAddr() + _, serverAdminPort := staticServerProxy.GetAdminAddr() + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertEnvoyPresentsCertURI(t, adminPort, libservice.StaticClientServiceName) + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPort, libservice.StaticServerServiceName) + _, assertionAdditionalResources, restartFn, err := tc.create(cluster, staticClientProxy) + require.NoError(t, err) // validate client and proxy is up and running - libassert.AssertContainerState(t, clientConnectProxy, "running") - - libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) + libassert.AssertContainerState(t, staticClientProxy, "running") + assertionAdditionalResources() + tc.extraAssertion(staticClientProxy) // Upgrade cluster, restart sidecars then begin service traffic validation - require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) - require.NoError(t, clientConnectProxy.Restart()) - require.NoError(t, serverConnectProxy.Restart()) - require.NoError(t, serverConnectProxyV1.Restart()) - require.NoError(t, serverConnectProxyV2.Restart()) + require.NoError(t, cluster.StandardUpgrade(t, context.Background(), targetVersion)) + require.NoError(t, staticClientProxy.Restart()) + require.NoError(t, staticServerProxy.Restart()) + restartFn() // POST upgrade validation; repeat client & proxy validation libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) - - // validate static-client proxy admin is up - _, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, statusCode, fmt.Sprintf("service cannot be reached %v", statusCode)) - - // validate static-server proxy admin is up - _, statusCode1, err := libassert.GetEnvoyOutput(serverAdminPort, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, statusCode1, fmt.Sprintf("service cannot be reached %v", statusCode1)) - - // validate static-server-v1 proxy admin is up - _, statusCode2, err := libassert.GetEnvoyOutput(serverAdminPortV1, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, statusCode2, fmt.Sprintf("service cannot be reached %v", statusCode2)) - - // validate static-server-v2 proxy admin is up - _, statusCode3, err := libassert.GetEnvoyOutput(serverAdminPortV2, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, statusCode3, fmt.Sprintf("service cannot be reached %v", statusCode3)) + libassert.AssertEnvoyRunning(t, adminPort) + libassert.AssertEnvoyRunning(t, serverAdminPort) // certs are valid - libassert.AssertEnvoyPresentsCertURI(t, adminPort, "static-client") - libassert.AssertEnvoyPresentsCertURI(t, serverAdminPort, "static-server") - libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server") - libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, "static-server") + libassert.AssertEnvoyPresentsCertURI(t, adminPort, libservice.StaticClientServiceName) + libassert.AssertEnvoyPresentsCertURI(t, serverAdminPort, libservice.StaticServerServiceName) - // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2") + assertionAdditionalResources() + tc.extraAssertion(staticClientProxy) } - for _, tc := range tcs { - t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), - func(t *testing.T) { - run(t, tc) - }) + targetVersion := libutils.TargetVersion + for _, oldVersion := range upgrade.UpgradeFromVersions { + for _, tc := range tcs { + t.Run(fmt.Sprintf("%s upgrade from %s to %s", tc.name, oldVersion, targetVersion), + func(t *testing.T) { + run(t, tc, oldVersion, targetVersion) + }) + } } } -// create 3 servers and 1 client -func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service, libservice.Service, libservice.Service) { +// createStaticClientAndServer creates a static-client and a static-server in the cluster +func createStaticClientAndServer(cluster *libcluster.Cluster) (libservice.Service, libservice.Service, error) { node := cluster.Agents[0] - client := node.GetClient() - serviceOpts := &libservice.ServiceOpts{ Name: libservice.StaticServerServiceName, ID: "static-server", @@ -150,35 +407,15 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic GRPCPort: 8079, } _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-server") - - serviceOptsV1 := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server-v1", - Meta: map[string]string{"version": "v1"}, - HTTPPort: 8081, - GRPCPort: 8078, + if err != nil { + return nil, nil, err } - _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-server") - - serviceOptsV2 := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server-v2", - Meta: map[string]string{"version": "v2"}, - HTTPPort: 8082, - GRPCPort: 8077, - } - _, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-server") // Create a client proxy instance with the server as an upstream clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + if err != nil { + return nil, nil, err + } - return serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy + return clientConnectProxy, serverConnectProxy, nil } diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go deleted file mode 100644 index f33d74d6c1da..000000000000 --- a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_onlypassing_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package upgrade - -import ( - "context" - "fmt" - "net/http" - "testing" - "time" - - "github.com/hashicorp/consul/api" - libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" - libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" - "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" - libutils "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" - "github.com/hashicorp/go-version" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestTrafficManagement_ServiceResolverSubsetOnlyPassing Summary -// This test starts up 2 servers and 1 client in the same datacenter. -// -// Steps: -// - Create a single agent cluster. -// - Create one static-server, 1 subset server 1 client and sidecars for all services, then register them with Consul -func TestTrafficManagement_ServiceResolverSubsetOnlyPassing(t *testing.T) { - t.Parallel() - - responseFormat := map[string]string{"format": "json"} - - type testcase struct { - oldversion string - targetVersion string - } - tcs := []testcase{ - { - oldversion: "1.13", - targetVersion: utils.TargetVersion, - }, - { - oldversion: "1.14", - targetVersion: utils.TargetVersion, - }, - } - - run := func(t *testing.T, tc testcase) { - buildOpts := &libcluster.BuildOptions{ - ConsulVersion: tc.oldversion, - Datacenter: "dc1", - InjectAutoEncryption: true, - } - // If version < 1.14 disable AutoEncryption - oldVersion, _ := version.NewVersion(tc.oldversion) - if oldVersion.LessThan(libutils.Version_1_14) { - buildOpts.InjectAutoEncryption = false - } - cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) - node := cluster.Agents[0] - - // Register service resolver - serviceResolver := &api.ServiceResolverConfigEntry{ - Kind: api.ServiceResolver, - Name: libservice.StaticServerServiceName, - DefaultSubset: "test", - Subsets: map[string]api.ServiceResolverSubset{ - "test": { - OnlyPassing: true, - }, - }, - ConnectTimeout: 120 * time.Second, - } - err := cluster.ConfigEntryWrite(serviceResolver) - require.NoError(t, err) - - serverConnectProxy, serverConnectProxyV1, clientConnectProxy := createServiceAndSubset(t, cluster) - - _, port := clientConnectProxy.GetAddr() - _, adminPort := clientConnectProxy.GetAdminAddr() - _, serverAdminPort := serverConnectProxy.GetAdminAddr() - _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() - - // Upgrade cluster, restart sidecars then begin service traffic validation - require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) - require.NoError(t, clientConnectProxy.Restart()) - require.NoError(t, serverConnectProxy.Restart()) - require.NoError(t, serverConnectProxyV1.Restart()) - - // force static-server-v1 into a warning state - err = node.GetClient().Agent().UpdateTTL("service:static-server-v1", "", "warn") - assert.NoError(t, err) - - // validate static-client is up and running - libassert.AssertContainerState(t, clientConnectProxy, "running") - libassert.HTTPServiceEchoes(t, "localhost", port, "") - - // validate static-client proxy admin is up - _, clientStatusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, clientStatusCode, fmt.Sprintf("service cannot be reached %v", clientStatusCode)) - - // validate static-server proxy admin is up - _, serverStatusCode, err := libassert.GetEnvoyOutput(serverAdminPort, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, serverStatusCode, fmt.Sprintf("service cannot be reached %v", serverStatusCode)) - - // validate static-server-v1 proxy admin is up - _, serverStatusCodeV1, err := libassert.GetEnvoyOutput(serverAdminPortV1, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, serverStatusCodeV1, fmt.Sprintf("service cannot be reached %v", serverStatusCodeV1)) - - // certs are valid - libassert.AssertEnvoyPresentsCertURI(t, adminPort, libservice.StaticClientServiceName) - libassert.AssertEnvoyPresentsCertURI(t, serverAdminPort, libservice.StaticServerServiceName) - libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, libservice.StaticServerServiceName) - - // ########################### - // ## with onlypassing=true - // assert only one static-server proxy is healthy - libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1) - - // static-client upstream should have 1 healthy endpoint for test.static-server - libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "HEALTHY", 1) - - // static-client upstream should have 1 unhealthy endpoint for test.static-server - libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "UNHEALTHY", 1) - - // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName) - - // ########################### - // ## with onlypassing=false - // revert to OnlyPassing=false by deleting the config - err = cluster.ConfigEntryDelete(serviceResolver) - require.NoError(t, err) - - // Consul health check assert only one static-server proxy is healthy when onlyPassing is false - libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, false, 2) - - // Although the service status is in warning state, when onlypassing is set to false Envoy - // health check returns all service instances with "warning" or "passing" state as Healthy enpoints - libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "HEALTHY", 2) - - // static-client upstream should have 0 unhealthy endpoint for static-server - libassert.AssertUpstreamEndpointStatus(t, adminPort, "static-server.default", "UNHEALTHY", 0) - } - - for _, tc := range tcs { - t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), - func(t *testing.T) { - run(t, tc) - }) - } -} - -// create 2 servers and 1 client -func createServiceAndSubset(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service, libservice.Service) { - node := cluster.Agents[0] - client := node.GetClient() - - serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: libservice.StaticServerServiceName, - HTTPPort: 8080, - GRPCPort: 8079, - } - _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) - - serviceOptsV1 := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server-v1", - Meta: map[string]string{"version": "v1"}, - HTTPPort: 8081, - GRPCPort: 8078, - Checks: libservice.Checks{ - Name: "main", - TTL: "30m", - }, - Connect: libservice.SidecarService{ - Port: 21011, - }, - } - _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecarWithChecks(node, serviceOptsV1) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) - - // Create a client proxy instance with the server as an upstream - clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) - - return serverConnectProxy, serverConnectProxyV1, clientConnectProxy -} diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go deleted file mode 100644 index 3ba808057ac9..000000000000 --- a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go +++ /dev/null @@ -1,230 +0,0 @@ -package upgrade - -import ( - "context" - "fmt" - "net/http" - "testing" - "time" - - "github.com/hashicorp/consul/api" - libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" - libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" - "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" - "github.com/hashicorp/go-version" - "github.com/stretchr/testify/require" - "gotest.tools/assert" -) - -// TestTrafficManagement_ServiceResolverSubsetRedirect Summary -// This test starts up 4 servers and 1 client in the same datacenter. -// -// Steps: -// - Create a single agent cluster. -// - Create 2 static-servers, 2 subset servers and 1 client and sidecars for all services, then register them with Consul -// - Validate traffic is successfully redirected from server 1 to sever2-v2 as defined in the service resolver -func TestTrafficManagement_ServiceResolverSubsetRedirect(t *testing.T) { - t.Parallel() - - type testcase struct { - oldversion string - targetVersion string - } - tcs := []testcase{ - { - oldversion: "1.13", - targetVersion: utils.TargetVersion, - }, - { - oldversion: "1.14", - targetVersion: utils.TargetVersion, - }, - } - - run := func(t *testing.T, tc testcase) { - buildOpts := &libcluster.BuildOptions{ - ConsulVersion: tc.oldversion, - Datacenter: "dc1", - InjectAutoEncryption: true, - } - // If version < 1.14 disable AutoEncryption - oldVersion, _ := version.NewVersion(tc.oldversion) - if oldVersion.LessThan(utils.Version_1_14) { - buildOpts.InjectAutoEncryption = false - } - cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) - node := cluster.Agents[0] - - // Register static-server service resolver - serviceResolver := &api.ServiceResolverConfigEntry{ - Kind: api.ServiceResolver, - Name: libservice.StaticServer2ServiceName, - Subsets: map[string]api.ServiceResolverSubset{ - "v1": { - Filter: "Service.Meta.version == v1", - }, - "v2": { - Filter: "Service.Meta.version == v2", - }, - }, - } - err := cluster.ConfigEntryWrite(serviceResolver) - require.NoError(t, err) - - // Register static-server-2 service resolver to redirect traffic - // from static-server to static-server-2-v2 - service2Resolver := &api.ServiceResolverConfigEntry{ - Kind: api.ServiceResolver, - Name: libservice.StaticServerServiceName, - Redirect: &api.ServiceResolverRedirect{ - Service: libservice.StaticServer2ServiceName, - ServiceSubset: "v2", - }, - } - err = cluster.ConfigEntryWrite(service2Resolver) - require.NoError(t, err) - - // register agent services - agentServices := setupServiceAndSubsets(t, cluster) - assertionFn, proxyRestartFn := agentServices.validateAgentServices(t) - _, port := agentServices.client.GetAddr() - _, adminPort := agentServices.client.GetAdminAddr() - - // validate static-client is up and running - libassert.AssertContainerState(t, agentServices.client, "running") - libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") - assertionFn() - - // Upgrade cluster, restart sidecars then begin service traffic validation - require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) - proxyRestartFn() - - libassert.AssertContainerState(t, agentServices.client, "running") - libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") - assertionFn() - - // assert 3 static-server instances are healthy - libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServer2ServiceName, false, 3) - libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server-2.default", "HEALTHY", 1) - } - - for _, tc := range tcs { - t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), - func(t *testing.T) { - run(t, tc) - }) - // test sometimes fails with error: could not start or join all agents: could not add container index 0: port not found - time.Sleep(1 * time.Second) - } -} - -func (s *registeredServices) validateAgentServices(t *testing.T) (func(), func()) { - var ( - responseFormat = map[string]string{"format": "json"} - servicePort = make(map[string]int) - proxyRestartFn func() - assertionFn func() - ) - - for serviceName, proxies := range s.services { - for _, proxy := range proxies { - _, adminPort := proxy.GetAdminAddr() - servicePort[serviceName] = adminPort - } - } - - assertionFn = func() { - // validate services proxy admin is up - for serviceName, adminPort := range servicePort { - _, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, statusCode, fmt.Sprintf("%s cannot be reached %v", serviceName, statusCode)) - - // certs are valid - libassert.AssertEnvoyPresentsCertURI(t, adminPort, serviceName) - } - } - - for _, serviceConnectProxy := range s.services { - for _, proxy := range serviceConnectProxy { - proxyRestartFn = func() { require.NoErrorf(t, proxy.Restart(), "%s", proxy.GetName()) } - } - } - return assertionFn, proxyRestartFn -} - -type registeredServices struct { - client libservice.Service - services map[string][]libservice.Service -} - -// create 3 servers and 1 client -func setupServiceAndSubsets(t *testing.T, cluster *libcluster.Cluster) *registeredServices { - node := cluster.Agents[0] - client := node.GetClient() - - // create static-servers and subsets - serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", - HTTPPort: 8080, - GRPCPort: 8079, - } - _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) - - serviceOpts2 := &libservice.ServiceOpts{ - Name: libservice.StaticServer2ServiceName, - ID: "static-server-2", - HTTPPort: 8081, - GRPCPort: 8078, - } - _, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) - - serviceOptsV1 := &libservice.ServiceOpts{ - Name: libservice.StaticServer2ServiceName, - ID: "static-server-2-v1", - Meta: map[string]string{"version": "v1"}, - HTTPPort: 8082, - GRPCPort: 8077, - } - _, server2ConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) - - serviceOptsV2 := &libservice.ServiceOpts{ - Name: libservice.StaticServer2ServiceName, - ID: "static-server-2-v2", - Meta: map[string]string{"version": "v2"}, - HTTPPort: 8083, - GRPCPort: 8076, - } - _, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) - - // Create a client proxy instance with the server as an upstream - clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) - - // return a map of all services created - tmpServices := map[string][]libservice.Service{} - tmpServices[libservice.StaticClientServiceName] = append(tmpServices[libservice.StaticClientServiceName], clientConnectProxy) - tmpServices[libservice.StaticServerServiceName] = append(tmpServices[libservice.StaticServerServiceName], serverConnectProxy) - tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxy) - tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV1) - tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV2) - - return ®isteredServices{ - client: clientConnectProxy, - services: tmpServices, - } -} diff --git a/test/integration/consul-container/test/upgrade/upgrade.go b/test/integration/consul-container/test/upgrade/upgrade.go new file mode 100644 index 000000000000..30bbdcc627c4 --- /dev/null +++ b/test/integration/consul-container/test/upgrade/upgrade.go @@ -0,0 +1,3 @@ +package upgrade + +var UpgradeFromVersions = []string{"1.13", "1.14"} From d4f51f70fc0370182961e8018212171f41ad3fb0 Mon Sep 17 00:00:00 2001 From: Tu Nguyen Date: Tue, 28 Feb 2023 10:00:00 -0800 Subject: [PATCH 058/421] udpate docs so they're sentence case, style guide (#16461) --- website/content/docs/agent/limits/index.mdx | 3 ++- website/content/docs/agent/limits/init-rate-limits.mdx | 2 +- .../docs/agent/limits/set-global-traffic-rate-limits.mdx | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/website/content/docs/agent/limits/index.mdx b/website/content/docs/agent/limits/index.mdx index 9d88f5f4b0c2..75fb4f1ac0e8 100644 --- a/website/content/docs/agent/limits/index.mdx +++ b/website/content/docs/agent/limits/index.mdx @@ -5,7 +5,8 @@ description: Rate limiting is a set of Consul server agent configurations that y --- -# Limit Traffic Rates Overview +# Limit traffic rates overview + This topic provides overview information about the traffic rates limits you can configure for Consul servers. ## Introduction diff --git a/website/content/docs/agent/limits/init-rate-limits.mdx b/website/content/docs/agent/limits/init-rate-limits.mdx index 5f9b5ac0580e..cb85d052a762 100644 --- a/website/content/docs/agent/limits/init-rate-limits.mdx +++ b/website/content/docs/agent/limits/init-rate-limits.mdx @@ -4,7 +4,7 @@ page_title: Initialize Rate Limit Settings description: Learn how to determins regular and peak loads in your network so that you can set the initial global rate limit configurations. --- -# Initialize Rate Limit Settings +# Initialize rate limit settings In order to set limits for traffic, you must first understand regular and peak loads in your network. We recommend completing the following steps to benchmark request rates in your environment so that you can implement limits appropriate for your applications. diff --git a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx index fc655833865c..369d1b7b5839 100644 --- a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx +++ b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx @@ -3,7 +3,8 @@ layout: docs page_title: Set a Global Limit on Traffic Rates description: Use global rate limits to prevent excessive rates of requests to Consul servers. --- -# Set a Global Limit on Traffic Rates + +# Set a global limit on traffic rates This topic describes how to configure rate limits for RPC and gRPC traffic to the Consul server. From 0fe36dd247f1a61e64681f275b6bda2a408a7726 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 10:05:09 -0800 Subject: [PATCH 059/421] backport of commit e2f570f13f6e61cad78f91970b2439381ff841f2 (#16403) Co-authored-by: Poonam Jadhav Co-authored-by: Tu Nguyen --- website/content/docs/agent/limits/init-rate-limits.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/agent/limits/init-rate-limits.mdx b/website/content/docs/agent/limits/init-rate-limits.mdx index cb85d052a762..320571845309 100644 --- a/website/content/docs/agent/limits/init-rate-limits.mdx +++ b/website/content/docs/agent/limits/init-rate-limits.mdx @@ -26,7 +26,7 @@ In order to set limits for traffic, you must first understand regular and peak l 1. Observe the logs and metrics for your application's typical cycle, such as a 24 hour period. Refer to [`log_file`](/consul/docs/agent/config/config-files#log_file) for information about where to retrieve logs. Call the [`/agent/metrics`](/consul/api-docs/agent#view-metrics) HTTP API endpoint and check the data for the following metrics: - - `rpc.rate_limit.exceeded.read` - - `rpc.rate_limit.exceeded.write` + - `rpc.rate_limit.exceeded` with value `global/read` for label `limit_type` + - `rpc.rate_limit.exceeded` with value `global/write` for label `limit_type` 1. If the limits are not reached, set the `mode` configuration to `enforcing`. Otherwise adjust and iterate limits. \ No newline at end of file From a6e7830c9f3f297e59a52e7801098e50e26a3b6a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 10:39:07 -0800 Subject: [PATCH 060/421] Backport of docs: Add backwards compatibility for Consul 1.14.x and consul-dataplane in the Envoy compat matrix into release/1.15.x (#16463) * backport of commit 640855e7554f965901c98fdbbdbd123b80c75b5d * backport of commit 4df972e79269797dc1ca1b6ca16b7dd6defb73a2 --------- Co-authored-by: David Yu --- website/content/docs/connect/proxies/envoy.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 3186b645c2f8..61ea0bf738c3 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -45,12 +45,12 @@ Consul supports **four major Envoy releases** at the beginning of each major Con ### Envoy and Consul Dataplane -Consul Dataplane is a feature introduced in Consul v1.14. Because each version of Consul Dataplane supports one specific version of Envoy, you must use the following versions of Consul, Consul Dataplane, and Envoy together. +The Consul dataplane component was introduced in Consul v1.14 as a way to manage Envoy proxies without the use of Consul clients. Each new minor version of Consul is released with a new minor version of Consul dataplane, which packages both Envoy and the `consul-dataplane` binary in a single container image. For backwards compatability reasons, each new minor version of Consul will also support the previous minor version of Consul dataplane to allow for seamless upgrades. In addition, each minor version of Consul will support the next minor version of Consul dataplane to allow for extended dataplane support via newer versions of Envoy. | Consul Version | Consul Dataplane Version (Bundled Envoy Version) | | ------------------- | ------------------------------------------------- | | 1.15.x | 1.1.x (Envoy 1.25.x), 1.0.x (Envoy 1.24.x) | -| 1.14.x | 1.0.x (Envoy 1.24.x) | +| 1.14.x | 1.1.x (Envoy 1.25.x), 1.0.x (Envoy 1.24.x) | ## Getting Started From e91cc8f16e0699e3190e2ee594c3adf5a36b3b21 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 11:12:03 -0800 Subject: [PATCH 061/421] Backport of cli: ensure acl token read -self works into release/1.15.x (#16459) * backport of commit f27d13de97986d73726224f7d553352e52e66300 * backport of commit 909f255680ce254fda0a8e1f35a7a3473ec97433 --------- Co-authored-by: R.B. Boyer --- .changelog/16445.txt | 3 ++ command/acl/token/read/token_read.go | 22 +++++------ command/acl/token/read/token_read_test.go | 47 +++++++++++++++++++++++ 3 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 .changelog/16445.txt diff --git a/.changelog/16445.txt b/.changelog/16445.txt new file mode 100644 index 000000000000..19745c6df99c --- /dev/null +++ b/.changelog/16445.txt @@ -0,0 +1,3 @@ +```release-note:bug +cli: ensure acl token read -self works +``` diff --git a/command/acl/token/read/token_read.go b/command/acl/token/read/token_read.go index 79ee10f4f702..0554ccaccb52 100644 --- a/command/acl/token/read/token_read.go +++ b/command/acl/token/read/token_read.go @@ -67,17 +67,6 @@ func (c *cmd) Run(args []string) int { return 1 } - tokenAccessor := c.tokenAccessorID - if tokenAccessor == "" { - if c.tokenID == "" { - c.UI.Error("Must specify the -accessor-id parameter") - return 1 - } else { - tokenAccessor = c.tokenID - c.UI.Warn("Use the -accessor-id parameter to specify token by Accessor ID") - } - } - client, err := c.http.APIClient() if err != nil { c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) @@ -87,6 +76,17 @@ func (c *cmd) Run(args []string) int { var t *api.ACLToken var expanded *api.ACLTokenExpanded if !c.self { + tokenAccessor := c.tokenAccessorID + if tokenAccessor == "" { + if c.tokenID == "" { + c.UI.Error("Must specify the -accessor-id parameter") + return 1 + } else { + tokenAccessor = c.tokenID + c.UI.Warn("Use the -accessor-id parameter to specify token by Accessor ID") + } + } + tok, err := acl.GetTokenAccessorIDFromPartial(client, tokenAccessor) if err != nil { c.UI.Error(fmt.Sprintf("Error determining token ID: %v", err)) diff --git a/command/acl/token/read/token_read_test.go b/command/acl/token/read/token_read_test.go index 505b15b02fa3..7988f9772ac9 100644 --- a/command/acl/token/read/token_read_test.go +++ b/command/acl/token/read/token_read_test.go @@ -116,3 +116,50 @@ func TestTokenReadCommand_JSON(t *testing.T) { err = json.Unmarshal([]byte(ui.OutputWriter.String()), &jsonOutput) require.NoError(t, err, "token unmarshalling error") } + +func TestTokenReadCommand_Self(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + a := agent.NewTestAgent(t, ` + primary_datacenter = "dc1" + acl { + enabled = true + tokens { + initial_management = "root" + } + }`) + + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + ui := cli.NewMockUi() + cmd := New(ui) + + // Create a token + client := a.Client() + + token, _, err := client.ACL().TokenCreate( + &api.ACLToken{Description: "test"}, + &api.WriteOptions{Token: "root"}, + ) + assert.NoError(t, err) + + args := []string{ + "-http-addr=" + a.HTTPAddr(), + "-token=" + token.SecretID, + "-self", + } + + code := cmd.Run(args) + assert.Equal(t, code, 0) + assert.Empty(t, ui.ErrorWriter.String()) + + output := ui.OutputWriter.String() + assert.Contains(t, output, fmt.Sprintf("test")) + assert.Contains(t, output, token.AccessorID) + assert.Contains(t, output, token.SecretID) +} From d63d822cdd17a3b9b6c0dcf150aa2724c64ce589 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 12:57:45 -0800 Subject: [PATCH 062/421] backport of commit e734b0c1df51831cd61e39097f0460275ce3bddb (#16468) Co-authored-by: R.B. Boyer --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 175f3ef55c04..36b2eacfa5a1 100644 --- a/version/version.go +++ b/version/version.go @@ -20,7 +20,7 @@ var ( //go:embed VERSION fullVersion string - Version, VersionPrerelease, _ = strings.Cut(fullVersion, "-") + Version, VersionPrerelease, _ = strings.Cut(strings.TrimSpace(fullVersion), "-") // https://semver.org/#spec-item-10 VersionMetadata = "" From 10b5250126edf3bb63071bc6fab53a6627c9db42 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 16:36:54 -0600 Subject: [PATCH 063/421] Backport of Gateway Test HTTPPathRewrite into release/1.15.x (#16466) * no-op commit due to failed cherry-picking * add http url path rewrite * add Mike's test back in * update kind to use api.APIGateway --------- Co-authored-by: temp Co-authored-by: Sarah Alsmiller Co-authored-by: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> --- .../test/gateways/gateway_endpoint_test.go | 130 ++++-- .../test/gateways/http_route_test.go | 440 ++++++++++++++++-- 2 files changed, 504 insertions(+), 66 deletions(-) diff --git a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go index e750a5b1fdc3..2aa81954f8d8 100644 --- a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go +++ b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go @@ -51,7 +51,7 @@ func TestAPIGatewayCreate(t *testing.T) { }, } _, _, err := client.ConfigEntries().Set(apiGateway, nil) - assert.NoError(t, err) + require.NoError(t, err) tcpRoute := &api.TCPRouteConfigEntry{ Kind: "tcp-route", @@ -70,32 +70,18 @@ func TestAPIGatewayCreate(t *testing.T) { } _, _, err = client.ConfigEntries().Set(tcpRoute, nil) - assert.NoError(t, err) + require.NoError(t, err) // Create a client proxy instance with the server as an upstream _, gatewayService := createServices(t, cluster, listenerPortOne) - //check statuses - gatewayReady := false - routeReady := false - //make sure the gateway/route come online - require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get("api-gateway", "api-gateway", nil) - assert.NoError(t, err) - apiEntry := entry.(*api.APIGatewayConfigEntry) - gatewayReady = isAccepted(apiEntry.Status.Conditions) - - e, _, err := client.ConfigEntries().Get("tcp-route", "api-gateway-route", nil) - assert.NoError(t, err) - routeEntry := e.(*api.TCPRouteConfigEntry) - routeReady = isBound(routeEntry.Status.Conditions) - - return gatewayReady && routeReady - }, time.Second*10, time.Second*1) + //make sure config entries have been properly created + checkGatewayConfigEntry(t, client, "api-gateway", "") + checkTCPRouteConfigEntry(t, client, "api-gateway-route", "") port, err := gatewayService.GetPort(listenerPortOne) - assert.NoError(t, err) + require.NoError(t, err) libassert.HTTPServiceEchoes(t, "localhost", port, "") } @@ -150,12 +136,64 @@ func createCluster(t *testing.T, ports ...int) *libcluster.Cluster { return cluster } +func createGateway(gatewayName string, protocol string, listenerPort int) *api.APIGatewayConfigEntry { + return &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: gatewayName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerPort, + Protocol: protocol, + }, + }, + } +} + +func checkGatewayConfigEntry(t *testing.T, client *api.Client, gatewayName string, namespace string) { + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace}) + require.NoError(t, err) + if entry == nil { + return false + } + apiEntry := entry.(*api.APIGatewayConfigEntry) + return isAccepted(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) +} + +func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) + require.NoError(t, err) + if entry == nil { + return false + } + + apiEntry := entry.(*api.HTTPRouteConfigEntry) + return isBound(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) +} + +func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.TCPRoute, routeName, &api.QueryOptions{Namespace: namespace}) + require.NoError(t, err) + if entry == nil { + return false + } + + apiEntry := entry.(*api.TCPRouteConfigEntry) + return isBound(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) +} + func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service { node := cluster.Agents[0] client := node.GetClient() // Create a service and proxy instance service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...) - assert.NoError(t, err) + require.NoError(t, err) libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy") libassert.CatalogServiceExists(t, client, serviceOpts.Name) @@ -183,15 +221,18 @@ func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (li return clientConnectProxy, gatewayService } -// checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances - type checkOptions struct { debug bool statusCode int testName string } -func checkRoute(t *testing.T, ip string, port int, path string, headers map[string]string, expected checkOptions) { +// checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances +func checkRoute(t *testing.T, port int, path string, headers map[string]string, expected checkOptions) { + ip := "localhost" + if expected.testName != "" { + t.Log("running " + expected.testName) + } const phrase = "hello" failer := func() *retry.Timer { @@ -199,17 +240,15 @@ func checkRoute(t *testing.T, ip string, port int, path string, headers map[stri } client := cleanhttp.DefaultClient() - url := fmt.Sprintf("http://%s:%d", ip, port) - if path != "" { - url += "/" + path - } + path = strings.TrimPrefix(path, "/") + url := fmt.Sprintf("http://%s:%d/%s", ip, port, path) retry.RunWith(failer(), t, func(r *retry.R) { t.Logf("making call to %s", url) reader := strings.NewReader(phrase) req, err := http.NewRequest("POST", url, reader) - assert.NoError(t, err) + require.NoError(t, err) headers["content-type"] = "text/plain" for k, v := range headers { @@ -248,3 +287,36 @@ func checkRoute(t *testing.T, ip string, port int, path string, headers map[stri }) } + +func checkRouteError(t *testing.T, ip string, port int, path string, headers map[string]string, expected string) { + failer := func() *retry.Timer { + return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} + } + + client := cleanhttp.DefaultClient() + url := fmt.Sprintf("http://%s:%d", ip, port) + + if path != "" { + url += "/" + path + } + + retry.RunWith(failer(), t, func(r *retry.R) { + t.Logf("making call to %s", url) + req, err := http.NewRequest("GET", url, nil) + assert.NoError(t, err) + + for k, v := range headers { + req.Header.Set(k, v) + + if k == "Host" { + req.Host = v + } + } + _, err = client.Do(req) + assert.Error(t, err) + + if expected != "" { + assert.ErrorContains(t, err, expected) + } + }) +} diff --git a/test/integration/consul-container/test/gateways/http_route_test.go b/test/integration/consul-container/test/gateways/http_route_test.go index 26f83ae92eca..61343f34950d 100644 --- a/test/integration/consul-container/test/gateways/http_route_test.go +++ b/test/integration/consul-container/test/gateways/http_route_test.go @@ -83,7 +83,7 @@ func TestHTTPRouteFlattening(t *testing.T) { } _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) - assert.NoError(t, err) + require.NoError(t, err) apiGateway := &api.APIGatewayConfigEntry{ Kind: "api-gateway", @@ -174,11 +174,11 @@ func TestHTTPRouteFlattening(t *testing.T) { } _, _, err = client.ConfigEntries().Set(apiGateway, nil) - assert.NoError(t, err) + require.NoError(t, err) _, _, err = client.ConfigEntries().Set(routeOne, nil) - assert.NoError(t, err) + require.NoError(t, err) _, _, err = client.ConfigEntries().Set(routeTwo, nil) - assert.NoError(t, err) + require.NoError(t, err) //create gateway service gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort) @@ -186,8 +186,290 @@ func TestHTTPRouteFlattening(t *testing.T) { libassert.CatalogServiceExists(t, client, gatewayName) //make sure config entries have been properly created + checkGatewayConfigEntry(t, client, gatewayName, namespace) + checkHTTPRouteConfigEntry(t, client, routeOneName, namespace) + checkHTTPRouteConfigEntry(t, client, routeTwoName, namespace) + + //gateway resolves routes + gatewayPort, err := gatewayService.GetPort(listenerPort) + require.NoError(t, err) + + //route 2 with headers + + //Same v2 path with and without header + checkRoute(t, gatewayPort, "/v2", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"}) + checkRoute(t, gatewayPort, "/v2", map[string]string{ + "Host": "test.foo", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"}) + + ////v1 path with the header + checkRoute(t, gatewayPort, "/check", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"}) + + checkRoute(t, gatewayPort, "/v2/path/value", map[string]string{ + "Host": "test.foo", + "x-v2": "v2", + }, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"}) + + //hit service 1 by hitting root path + checkRoute(t, gatewayPort, "", map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"}) + + //hit service 1 by hitting v2 path with v1 hostname + checkRoute(t, gatewayPort, "/v2", map[string]string{ + "Host": "test.example", + }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"}) + +} + +func TestHTTPRoutePathRewrite(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + //infrastructure set up + listenerPort := 6001 + //create cluster + cluster := createCluster(t, listenerPort) + client := cluster.Agents[0].GetClient() + fooStatusCode := 400 + barStatusCode := 201 + fooPath := "/v1/foo" + barPath := "/v1/bar" + + fooService := createService(t, cluster, &libservice.ServiceOpts{ + Name: "foo", + ID: "foo", + HTTPPort: 8080, + GRPCPort: 8081, + }, []string{ + //customizes response code so we can distinguish between which service is responding + "-echo-debug-path", fooPath, + "-echo-server-default-params", fmt.Sprintf("status=%d", fooStatusCode), + }) + barService := createService(t, cluster, &libservice.ServiceOpts{ + Name: "bar", + ID: "bar", + //TODO we can potentially get conflicts if these ports are the same + HTTPPort: 8079, + GRPCPort: 8078, + }, []string{ + "-echo-debug-path", barPath, + "-echo-server-default-params", fmt.Sprintf("status=%d", barStatusCode), + }, + ) + + namespace := getNamespace() + gatewayName := randomName("gw", 16) + invalidRouteName := randomName("route", 16) + validRouteName := randomName("route", 16) + fooUnrewritten := "/foo" + barUnrewritten := "/bar" + + //write config entries + proxyDefaults := &api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, + Namespace: namespace, + Config: map[string]interface{}{ + "protocol": "http", + }, + } + + _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) + require.NoError(t, err) + + apiGateway := createGateway(gatewayName, "http", listenerPort) + + fooRoute := &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: invalidRouteName, + Parents: []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayName, + Namespace: namespace, + }, + }, + Hostnames: []string{ + "test.foo", + }, + Namespace: namespace, + Rules: []api.HTTPRouteRule{ + { + Filters: api.HTTPFilters{ + URLRewrite: &api.URLRewrite{ + Path: fooPath, + }, + }, + Services: []api.HTTPService{ + { + Name: fooService.GetServiceName(), + Namespace: namespace, + }, + }, + Matches: []api.HTTPMatch{ + { + Path: api.HTTPPathMatch{ + Match: api.HTTPPathMatchPrefix, + Value: fooUnrewritten, + }, + }, + }, + }, + }, + } + + barRoute := &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: validRouteName, + Parents: []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayName, + Namespace: namespace, + }, + }, + Hostnames: []string{ + "test.foo", + }, + Namespace: namespace, + Rules: []api.HTTPRouteRule{ + { + Filters: api.HTTPFilters{ + URLRewrite: &api.URLRewrite{ + Path: barPath, + }, + }, + Services: []api.HTTPService{ + { + Name: barService.GetServiceName(), + Namespace: namespace, + }, + }, + Matches: []api.HTTPMatch{ + { + Path: api.HTTPPathMatch{ + Match: api.HTTPPathMatchPrefix, + Value: barUnrewritten, + }, + }, + }, + }, + }, + } + + _, _, err = client.ConfigEntries().Set(apiGateway, nil) + require.NoError(t, err) + _, _, err = client.ConfigEntries().Set(fooRoute, nil) + require.NoError(t, err) + _, _, err = client.ConfigEntries().Set(barRoute, nil) + require.NoError(t, err) + + //create gateway service + gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, gatewayName) + + //make sure config entries have been properly created + checkGatewayConfigEntry(t, client, gatewayName, namespace) + checkHTTPRouteConfigEntry(t, client, invalidRouteName, namespace) + checkHTTPRouteConfigEntry(t, client, validRouteName, namespace) + + gatewayPort, err := gatewayService.GetPort(listenerPort) + require.NoError(t, err) + + //TODO these were the assertions we had in the original test. potentially would want more test cases + + //NOTE: Hitting the debug path code overrides default expected value + debugExpectedStatusCode := 200 + + //hit foo, making sure path is being rewritten by hitting the debug page + checkRoute(t, gatewayPort, fooUnrewritten, map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "foo service"}) + //make sure foo is being sent to proper service + checkRoute(t, gatewayPort, fooUnrewritten+"/foo", map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: false, statusCode: fooStatusCode, testName: "foo service"}) + + //hit bar, making sure its been rewritten + checkRoute(t, gatewayPort, barUnrewritten, map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "bar service"}) + + //hit bar, making sure its being sent to the proper service + checkRoute(t, gatewayPort, barUnrewritten+"/bar", map[string]string{ + "Host": "test.foo", + }, checkOptions{debug: false, statusCode: barStatusCode, testName: "bar service"}) + +} + +func TestHTTPRouteParentRefChange(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + // infrastructure set up + address := "localhost" + + listenerOnePort := 6000 + listenerTwoPort := 6001 + + // create cluster and service + cluster := createCluster(t, listenerOnePort, listenerTwoPort) + client := cluster.Agents[0].GetClient() + service := createService(t, cluster, &libservice.ServiceOpts{ + Name: "service", + ID: "service", + HTTPPort: 8080, + GRPCPort: 8079, + }, []string{}) + + // getNamespace() should always return an empty string in Consul OSS + namespace := getNamespace() + gatewayOneName := randomName("gw1", 16) + gatewayTwoName := randomName("gw2", 16) + routeName := randomName("route", 16) + + // write config entries + proxyDefaults := &api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, + Namespace: namespace, + Config: map[string]interface{}{ + "protocol": "http", + }, + } + _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) + assert.NoError(t, err) + + // create gateway config entry + gatewayOne := &api.APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: gatewayOneName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerOnePort, + Protocol: "http", + Hostname: "test.foo", + }, + }, + } + _, _, err = client.ConfigEntries().Set(gatewayOne, nil) + assert.NoError(t, err) require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace}) + entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace}) assert.NoError(t, err) if entry == nil { return false @@ -197,62 +479,146 @@ func TestHTTPRouteFlattening(t *testing.T) { return isAccepted(apiEntry.Status.Conditions) }, time.Second*10, time.Second*1) + // create gateway service + gatewayOneService, err := libservice.NewGatewayService(context.Background(), gatewayOneName, "api", cluster.Agents[0], listenerOnePort) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, gatewayOneName) + + // create gateway config entry + gatewayTwo := &api.APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: gatewayTwoName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerTwoPort, + Protocol: "http", + Hostname: "test.example", + }, + }, + } + _, _, err = client.ConfigEntries().Set(gatewayTwo, nil) + assert.NoError(t, err) require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeOneName, &api.QueryOptions{Namespace: namespace}) + entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace}) assert.NoError(t, err) if entry == nil { return false } - - apiEntry := entry.(*api.HTTPRouteConfigEntry) + apiEntry := entry.(*api.APIGatewayConfigEntry) t.Log(entry) - return isBound(apiEntry.Status.Conditions) + return isAccepted(apiEntry.Status.Conditions) }, time.Second*10, time.Second*1) + // create gateway service + gatewayTwoService, err := libservice.NewGatewayService(context.Background(), gatewayTwoName, "api", cluster.Agents[0], listenerTwoPort) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, gatewayTwoName) + + // create route to service, targeting first gateway + route := &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: routeName, + Parents: []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayOneName, + Namespace: namespace, + }, + }, + Hostnames: []string{ + "test.foo", + "test.example", + }, + Namespace: namespace, + Rules: []api.HTTPRouteRule{ + { + Services: []api.HTTPService{ + { + Name: service.GetServiceName(), + Namespace: namespace, + }, + }, + Matches: []api.HTTPMatch{ + { + Path: api.HTTPPathMatch{ + Match: api.HTTPPathMatchPrefix, + Value: "/", + }, + }, + }, + }, + }, + } + _, _, err = client.ConfigEntries().Set(route, nil) + assert.NoError(t, err) require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeTwoName, nil) + entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) assert.NoError(t, err) if entry == nil { return false } apiEntry := entry.(*api.HTTPRouteConfigEntry) - return isBound(apiEntry.Status.Conditions) + t.Log(entry) + + // check if bound only to correct gateway + return len(apiEntry.Parents) == 1 && + apiEntry.Parents[0].Name == gatewayOneName && + isBound(apiEntry.Status.Conditions) }, time.Second*10, time.Second*1) - //gateway resolves routes - ip := "localhost" - gatewayPort, err := gatewayService.GetPort(listenerPort) + // fetch gateway listener ports + gatewayOnePort, err := gatewayOneService.GetPort(listenerOnePort) + assert.NoError(t, err) + gatewayTwoPort, err := gatewayTwoService.GetPort(listenerTwoPort) assert.NoError(t, err) - //Same v2 path with and without header - checkRoute(t, ip, gatewayPort, "v2", map[string]string{ - "Host": "test.foo", - "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"}) - checkRoute(t, ip, gatewayPort, "v2", map[string]string{ + // hit service by requesting root path + // TODO: testName field in checkOptions struct looked to be unused, is it needed? + checkRoute(t, gatewayOnePort, "", map[string]string{ "Host": "test.foo", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"}) + }, checkOptions{debug: false, statusCode: 200}) - ////v1 path with the header - checkRoute(t, ip, gatewayPort, "check", map[string]string{ - "Host": "test.foo", - "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"}) + // check that second gateway does not resolve service + checkRouteError(t, address, gatewayTwoPort, "", map[string]string{ + "Host": "test.example", + }, "") - checkRoute(t, ip, gatewayPort, "v2/path/value", map[string]string{ - "Host": "test.foo", - "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"}) + // swtich route target to second gateway + route.Parents = []api.ResourceReference{ + { + Kind: api.APIGateway, + Name: gatewayTwoName, + Namespace: namespace, + }, + } + _, _, err = client.ConfigEntries().Set(route, nil) + assert.NoError(t, err) + require.Eventually(t, func() bool { + entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) + assert.NoError(t, err) + if entry == nil { + return false + } - //hit service 1 by hitting root path - checkRoute(t, ip, gatewayPort, "", map[string]string{ - "Host": "test.foo", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"}) + apiEntry := entry.(*api.HTTPRouteConfigEntry) + t.Log(apiEntry) + t.Log(fmt.Sprintf("%#v", apiEntry)) - //hit service 1 by hitting v2 path with v1 hostname - checkRoute(t, ip, gatewayPort, "v2", map[string]string{ + // check if bound only to correct gateway + return len(apiEntry.Parents) == 1 && + apiEntry.Parents[0].Name == gatewayTwoName && + isBound(apiEntry.Status.Conditions) + }, time.Second*10, time.Second*1) + + // hit service by requesting root path on other gateway with different hostname + checkRoute(t, gatewayTwoPort, "", map[string]string{ "Host": "test.example", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"}) + }, checkOptions{debug: false, statusCode: 200}) + // check that first gateway has stopped resolving service + checkRouteError(t, address, gatewayOnePort, "", map[string]string{ + "Host": "test.foo", + }, "") } From 159b132f1772caedf8ed05a6f6f869e0c4e724a6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 16:46:51 -0600 Subject: [PATCH 064/421] backport of commit de8b971a59d48a7588d9e08cd4c05b19a702c8ef (#16474) Co-authored-by: skpratt --- website/content/docs/enterprise/license/faq.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/content/docs/enterprise/license/faq.mdx b/website/content/docs/enterprise/license/faq.mdx index 02841e9c6cfa..4221762b2dfa 100644 --- a/website/content/docs/enterprise/license/faq.mdx +++ b/website/content/docs/enterprise/license/faq.mdx @@ -74,6 +74,8 @@ after license expiration and is defined in ~> **Starting with Consul 1.14, and patch releases 1.13.3 and 1.12.6, Consul will support non-terminating licenses**: Please contact your support representative for more details on non-terminating licenses. + An expired license will not allow Consul versions released after the expiration date to run. + It will not be possible to upgrade to a new version of Consul released after license expiration. ## Q: Does this affect client agents? From 36775cc158b55936c1b4d16cbc166794547495d6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 18:48:51 -0600 Subject: [PATCH 065/421] Backport of Docs/services refactor docs day 122022 into release/1.15.x (#16470) * backport of commit 1c0ec4721f3f8b5ef72f5b47b032929b101a5370 * Docs/services refactor docs day 122022 (#16103) * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * fixed cross-links between new topics * updated links to the new services pages * fixed bad links in scale file * tweaks to titles and phrasing * fixed typo in checks.mdx * started updating the conf ref to latest template * update SD conf ref to match latest CT standard * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> * remove previous version of the checks page * fixed cross-links * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> --------- Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> --------- Co-authored-by: trujillo-adam Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> --- website/content/api-docs/agent/check.mdx | 3 +- website/content/api-docs/agent/service.mdx | 40 +- website/content/api-docs/api-structure.mdx | 2 +- website/content/api-docs/catalog.mdx | 22 +- .../content/api-docs/features/consistency.mdx | 2 +- website/content/api-docs/query.mdx | 7 +- .../content/commands/services/deregister.mdx | 16 +- .../content/commands/services/register.mdx | 20 +- .../content/docs/agent/config/cli-flags.mdx | 2 +- .../docs/agent/config/config-files.mdx | 46 +- website/content/docs/agent/index.mdx | 2 +- .../docs/architecture/anti-entropy.mdx | 11 +- website/content/docs/architecture/index.mdx | 2 +- website/content/docs/architecture/scale.mdx | 6 +- .../connect/cluster-peering/tech-specs.mdx | 2 +- .../config-entries/ingress-gateway.mdx | 7 +- .../config-entries/service-defaults.mdx | 1194 ++++++++++++++++- .../content/docs/connect/configuration.mdx | 21 +- .../docs/connect/gateways/ingress-gateway.mdx | 4 +- website/content/docs/connect/native/index.mdx | 10 +- .../docs/connect/observability/index.mdx | 20 +- .../content/docs/connect/proxies/index.mdx | 28 +- .../connect/proxies/managed-deprecated.mdx | 278 ---- .../docs/connect/registration/index.mdx | 7 +- .../registration/service-registration.mdx | 18 +- .../connect/registration/sidecar-service.mdx | 28 +- .../consul-vs-other/dns-tools-compare.mdx | 2 +- website/content/docs/discovery/checks.mdx | 885 ------------ website/content/docs/discovery/dns.mdx | 594 -------- website/content/docs/discovery/services.mdx | 701 ---------- .../docs/ecs/configuration-reference.mdx | 2 +- website/content/docs/ecs/manual/install.mdx | 10 +- .../docs/enterprise/admin-partitions.mdx | 2 +- website/content/docs/intro/index.mdx | 2 +- .../usage/establish-peering.mdx | 2 +- website/content/docs/k8s/dns.mdx | 2 +- website/content/docs/k8s/service-sync.mdx | 2 +- website/content/docs/nia/configuration.mdx | 2 +- .../docs/release-notes/consul/v1_13_x.mdx | 2 +- .../content/docs/security/acl/acl-rules.mdx | 13 +- .../content/docs/security/acl/acl-tokens.mdx | 2 +- .../checks-configuration-reference.mdx | 55 + .../services-configuration-overview.mdx | 28 + .../services-configuration-reference.mdx | 654 +++++++++ .../services/discovery/dns-configuration.mdx | 76 ++ .../discovery/dns-dynamic-lookups.mdx | 100 ++ .../docs/services/discovery/dns-overview.mdx | 41 + .../services/discovery/dns-static-lookups.mdx | 357 +++++ website/content/docs/services/services.mdx | 39 + .../content/docs/services/usage/checks.mdx | 592 ++++++++ .../docs/services/usage/define-services.mdx | 366 +++++ .../usage/register-services-checks.mdx | 67 + .../troubleshoot/troubleshoot-services.mdx | 4 +- .../docs/upgrading/upgrade-specific.mdx | 6 +- website/data/docs-nav-data.json | 70 +- 55 files changed, 3767 insertions(+), 2709 deletions(-) delete mode 100644 website/content/docs/connect/proxies/managed-deprecated.mdx delete mode 100644 website/content/docs/discovery/checks.mdx delete mode 100644 website/content/docs/discovery/dns.mdx delete mode 100644 website/content/docs/discovery/services.mdx create mode 100644 website/content/docs/services/configuration/checks-configuration-reference.mdx create mode 100644 website/content/docs/services/configuration/services-configuration-overview.mdx create mode 100644 website/content/docs/services/configuration/services-configuration-reference.mdx create mode 100644 website/content/docs/services/discovery/dns-configuration.mdx create mode 100644 website/content/docs/services/discovery/dns-dynamic-lookups.mdx create mode 100644 website/content/docs/services/discovery/dns-overview.mdx create mode 100644 website/content/docs/services/discovery/dns-static-lookups.mdx create mode 100644 website/content/docs/services/services.mdx create mode 100644 website/content/docs/services/usage/checks.mdx create mode 100644 website/content/docs/services/usage/define-services.mdx create mode 100644 website/content/docs/services/usage/register-services-checks.mdx diff --git a/website/content/api-docs/agent/check.mdx b/website/content/api-docs/agent/check.mdx index d29f0de8fc60..03364b6aeb31 100644 --- a/website/content/api-docs/agent/check.mdx +++ b/website/content/api-docs/agent/check.mdx @@ -6,8 +6,7 @@ description: The /agent/check endpoints interact with checks on the local agent # Check - Agent HTTP API -Consul's health check capabilities are described in the -[health checks overview](/consul/docs/discovery/checks). +Refer to [Define Health Checks](/consul/docs/services/usage/checks) for information about Consul health check capabilities. The `/agent/check` endpoints interact with health checks managed by the local agent in Consul. These should not be confused with checks in the catalog. diff --git a/website/content/api-docs/agent/service.mdx b/website/content/api-docs/agent/service.mdx index a232c4491a3b..38eedad35362 100644 --- a/website/content/api-docs/agent/service.mdx +++ b/website/content/api-docs/agent/service.mdx @@ -170,6 +170,8 @@ The table below shows this endpoint's support for ### Sample Request +The following example request calls the `web-sidecar-proxy` service: + ```shell-session $ curl \ http://127.0.0.1:8500/v1/agent/service/web-sidecar-proxy @@ -177,6 +179,11 @@ $ curl \ ### Sample Response +The response contains the fields specified in the [service +definition](/consul/docs/services/configuration/services-configuration-reference), but it includes an extra `ContentHash` field that contains the [hash-based blocking +query](/consul/api-docs/features/blocking#hash-based-blocking-queries) hash for the result. The +same hash is also present in `X-Consul-ContentHash`. + ```json { "Kind": "connect-proxy", @@ -227,12 +234,6 @@ $ curl \ } ``` -The response has the same structure as the [service -definition](/consul/docs/discovery/services) with one extra field `ContentHash` which -contains the [hash-based blocking -query](/consul/api-docs/features/blocking#hash-based-blocking-queries) hash for the result. The -same hash is also present in `X-Consul-ContentHash`. - ## Get local service health Retrieve an aggregated state of service(s) on the local agent by name. @@ -599,21 +600,18 @@ The corresponding CLI command is [`consul services register`](/consul/commands/s ### JSON Request Body Schema -Note that this endpoint, unlike most also [supports `snake_case`](/consul/docs/discovery/services#service-definition-parameter-case) -service definition keys for compatibility with the config file format. +The `/agent/service/register` endpoint supports camel case and _snake case_ for service definition keys. Snake case is a convention in which keys with two or more words have underscores between them. Snake case ensures compatibility with the agent configuration file format. - `Name` `(string: )` - Specifies the logical name of the service. Many service instances may share the same logical service name. We recommend using - [valid DNS labels](https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_hostnames) - for [compatibility with external DNS](/consul/docs/discovery/services#service-and-tag-names-with-dns). + valid DNS labels for service definition names. Refer to the Internet Engineering Task Force's [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123#page-72) for additional information. Service names that conform to standard usage ensures compatibility with external DNSs. Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference#name) for additional information. - `ID` `(string: "")` - Specifies a unique ID for this service. This must be unique per _agent_. This defaults to the `Name` parameter if not provided. - `Tags` `(array: nil)` - Specifies a list of tags to assign to the - service. These tags can be used for later filtering and are exposed via the APIs. - We recommend using [valid DNS labels](https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_hostnames) - for [compatibility with external DNS](/consul/docs/discovery/services#service-and-tag-names-with-dns) + service. Tags enable you to filter when querying for the services and are exposed in Consul APIs. We recommend using + valid DNS labels for tags. Refer to the Internet Engineering Task Force's [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123#page-72) for additional information. Tags that conform to standard usage ensures compatibility with external DNSs. Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference#tags) for additional information. - `Address` `(string: "")` - Specifies the address of the service. If not provided, the agent's address is used as the address for the service during @@ -676,19 +674,15 @@ service definition keys for compatibility with the config file format. service's port _and_ the tags would revert to the original value and all modifications would be lost. -- `Weights` `(Weights: nil)` - Specifies weights for the service. Please see the - [service documentation](/consul/docs/discovery/services) for more information about - weights. If this field is not provided weights will default to +- `Weights` `(Weights: nil)` - Specifies weights for the service. Refer to + [Services Configuration Reference](/consul/docs/services/configuraiton/services-configuration-reference#weights) for additional information. Default is `{"Passing": 1, "Warning": 1}`. - It is important to note that this applies only to the locally registered - service. If you have multiple nodes all registering the same service their - `EnableTagOverride` configuration and all other service configuration items - are independent of one another. Updating the tags for the service registered - on one node is independent of the same service (by name) registered on - another node. If `EnableTagOverride` is not specified the default value is + Weights only apply to the locally registered service. + If multiple nodes register the same service, each node implements `EnableTagOverride` and other service configuration items independently. Updating the tags for the service registered + on one node does not necessarily update the same tags on services with the same name registered on another node. If `EnableTagOverride` is not specified the default value is `false`. See [anti-entropy syncs](/consul/docs/architecture/anti-entropy) for - more info. + additional information. #### Connect Structure diff --git a/website/content/api-docs/api-structure.mdx b/website/content/api-docs/api-structure.mdx index 655399e79329..7a190894129a 100644 --- a/website/content/api-docs/api-structure.mdx +++ b/website/content/api-docs/api-structure.mdx @@ -89,7 +89,7 @@ However, we generally recommend using resource names that don't require URL-enco Depending on the validation that Consul applies to a resource name, Consul may still reject a request if it considers the resource name invalid for that endpoint. And even if Consul considers the resource name valid, it may degrade other functionality, -such as failed [DNS lookups](/consul/docs/discovery/dns) +such as failed [DNS lookups](/consul/docs/services/discovery/dns-overview) for nodes or services with names containing invalid DNS characters. This HTTP API capability also allows the diff --git a/website/content/api-docs/catalog.mdx b/website/content/api-docs/catalog.mdx index b9a8c7bbd745..2b470dd57886 100644 --- a/website/content/api-docs/catalog.mdx +++ b/website/content/api-docs/catalog.mdx @@ -55,15 +55,16 @@ The table below shows this endpoint's support for - `NodeMeta` `(map: nil)` - Specifies arbitrary KV metadata pairs for filtering purposes. -- `Service` `(Service: nil)` - Specifies to register a service. If `ID` is not - provided, it will be defaulted to the value of the `Service.Service` property. - Only one service with a given `ID` may be present per node. We recommend using - [valid DNS labels](https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_hostnames) - for service definition names for [compatibility with external DNS](/consul/docs/discovery/services#service-and-tag-names-with-dns). - The service `Tags`, `Address`, `Meta`, and `Port` fields are all optional. For more - information about these fields and the implications of setting them, - see the [Service - Agent API](/consul/api-docs/agent/service) page - as registering services differs between using this or the Services Agent endpoint. +- `Service` `(Service: nil)` - Contains an object the specifies the service to register. The the `Service.Service` field is required. If `Service.ID` is not provided, the default is the `Service.Service`. + You can only specify one service with a given `ID` per node. We recommend using + valid DNS labels for service definition names. Refer to the Internet Engineering Task Force's [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123#page-72) for additional information. Service names that conform to standard usage ensures compatibility with external DNSs. Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference#name) for additional information. + The following fields are optional: + - `Tags` + - `Address` + - `Meta` + - `Port` + + For additional information configuring services, refer to [Service - Agent API](/consul/api-docs/agent/service). The `/catalog` endpoint had different behaviors than the `/services` endpoint. - `Check` `(Check: nil)` - Specifies to register a check. The register API manipulates the health check entry in the Catalog, but it does not setup the @@ -78,8 +79,7 @@ The table below shows this endpoint's support for treated as a service level health check, instead of a node level health check. The `Status` must be one of `passing`, `warning`, or `critical`. - The `Definition` field can be provided with details for a TCP or HTTP health - check. For more information, see the [Health Checks](/consul/docs/discovery/checks) page. + You can provide defaults for TCP and HTTP health checks to the `Definition` field. Refer to [Health Checks](/consul/docs/services/usage/checks) for additional information. Multiple checks can be provided by replacing `Check` with `Checks` and sending an array of `Check` objects. diff --git a/website/content/api-docs/features/consistency.mdx b/website/content/api-docs/features/consistency.mdx index bcd7d621d65b..746b062ab4c6 100644 --- a/website/content/api-docs/features/consistency.mdx +++ b/website/content/api-docs/features/consistency.mdx @@ -131,7 +131,7 @@ The following diagrams show the cross-datacenter request paths when Consul serve ### Consul DNS Queries -When DNS queries are issued to [Consul's DNS interface](/consul/docs/discovery/dns), +When DNS queries are issued to [Consul's DNS interface](/consul/docs/services/discovery/dns-overview), Consul uses the `stale` consistency mode by default when interfacing with its underlying Consul service discovery HTTP APIs ([Catalog](/consul/api-docs/catalog), [Health](/consul/api-docs/health), and [Prepared Query](/consul/api-docs/query)). diff --git a/website/content/api-docs/query.mdx b/website/content/api-docs/query.mdx index b3e2dcc2525a..09aa867c479f 100644 --- a/website/content/api-docs/query.mdx +++ b/website/content/api-docs/query.mdx @@ -9,10 +9,9 @@ description: The /query endpoints manage and execute prepared queries in Consul. The `/query` endpoints create, update, destroy, and execute prepared queries. Prepared queries allow you to register a complex service query and then execute -it later via its ID or name to get a set of healthy nodes that provide a given -service. This is particularly useful in combination with Consul's -[DNS Interface](/consul/docs/discovery/dns#prepared-query-lookups) as it allows for much richer queries than -would be possible given the limited entry points exposed by DNS. +it later by specifying the query ID or name. Consul returns a set of healthy nodes that provide a given +service. Refer to +[Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for additional information. Check the [Geo Failover tutorial](/consul/tutorials/developer-discovery/automate-geo-failover) for details and examples for using prepared queries to implement geo failover for services. diff --git a/website/content/commands/services/deregister.mdx b/website/content/commands/services/deregister.mdx index 5cc774422c0a..79ea7cba27ba 100644 --- a/website/content/commands/services/deregister.mdx +++ b/website/content/commands/services/deregister.mdx @@ -13,23 +13,19 @@ Corresponding HTTP API Endpoint: [\[PUT\] /v1/agent/service/deregister/:service_ The `services deregister` command deregisters a service with the local agent. Note that this command can only deregister services that were registered -with the agent specified (defaults to the local agent) and is meant to -be paired with `services register`. +with the agent specified and is intended to be paired with `services register`. +By default, the command deregisters services on the local agent. -This is just one method for service deregistration. If the service was -registered with a configuration file, then deleting that file and -[reloading](/consul/commands/reload) Consul is the correct method to -deregister. See [Service Definition](/consul/docs/discovery/services) for more -information about registering services generally. +We recommend deregistering services registered with a configuration file by deleting the file and [reloading](/consul/commands/reload) Consul. Refer to [Services Overview](/consul/docs/services/services) for additional information about services. -The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of -[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) -are not supported from commands, but may be from the corresponding HTTP endpoint. +The following table shows the [ACLs](/consul/api-docs/api-structure#authentication) required to run the `consul services deregister` command: | ACL Required | | --------------- | | `service:write` | +You cannot use the Consul command line to configure [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching), you can configure them from the corresponding HTTP endpoint. + ## Usage Usage: `consul services deregister [options] [FILE...]` diff --git a/website/content/commands/services/register.mdx b/website/content/commands/services/register.mdx index 8e3a9d1333e9..cefa2359230b 100644 --- a/website/content/commands/services/register.mdx +++ b/website/content/commands/services/register.mdx @@ -14,24 +14,16 @@ Corresponding HTTP API Endpoint: [\[PUT\] /v1/agent/service/register](/consul/ap The `services register` command registers a service with the local agent. This command returns after registration and must be paired with explicit service deregistration. This command simplifies service registration from -scripts, in dev mode, etc. +scripts. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for information about other service registeration methods. -This is just one method of service registration. Services can also be -registered by placing a [service definition](/consul/docs/discovery/services) -in the Consul agent configuration directory and issuing a -[reload](/consul/commands/reload). This approach is easiest for -configuration management systems that other systems that have access to -the configuration directory. Clients may also use the -[HTTP API](/consul/api-docs/agent/service) directly. - -The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of -[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) -are not supported from commands, but may be from the corresponding HTTP endpoint. +The following table shows the [ACLs](/consul/api-docs/api-structure#authentication) required to use the `consul services register` command: | ACL Required | | --------------- | | `service:write` | +You cannot use the Consul command line to configure [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching), you can configure them from the corresponding HTTP endpoint. + ## Usage Usage: `consul services register [options] [FILE...]` @@ -65,9 +57,7 @@ The flags below should only be set if _no arguments_ are given. If no arguments are given, the flags below can be used to register a single service. -Note that the behavior of each of the fields below is exactly the same -as when constructing a standard [service definition](/consul/docs/discovery/services). -Please refer to that documentation for full details. +The following fields specify identical parameters in a standard service definition file. Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference) for details about each configuration option. - `-id` - The ID of the service. This will default to `-name` if not set. diff --git a/website/content/docs/agent/config/cli-flags.mdx b/website/content/docs/agent/config/cli-flags.mdx index ebcdb4c0764b..3cdfd6fe91b5 100644 --- a/website/content/docs/agent/config/cli-flags.mdx +++ b/website/content/docs/agent/config/cli-flags.mdx @@ -92,7 +92,7 @@ information. only the given `-encrypt` key will be available on startup. This defaults to false. - `-enable-script-checks` ((#\_enable_script_checks)) This controls whether - [health checks that execute scripts](/consul/docs/discovery/checks) are enabled on this + [health checks that execute scripts](/consul/docs/services/usage/checks) are enabled on this agent, and defaults to `false` so operators must opt-in to allowing these. This was added in Consul 0.9.0. diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 2992f562af00..705d0e837802 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -7,6 +7,10 @@ description: >- # Agents Configuration File Reference ((#configuration_files)) +This topic describes the parameters for configuring Consul agents. For information about how to start Consul agents, refer to [Starting the Consul Agent](/consul/docs/agent#starting-the-consul-agent). + +## Overview + You can create one or more files to configure the Consul agent on startup. We recommend grouping similar configurations into separate files, such as ACL parameters, to make it easier to manage configuration changes. Using external files may be easier than @@ -18,13 +22,6 @@ easily readable and editable by both humans and computers. JSON formatted configuration consists of a single JSON object with multiple configuration keys specified within it. -Configuration files are used for more than just setting up the agent. -They are also used to provide check and service definitions that -announce the availability of system servers to the rest of the cluster. -These definitions are documented separately under [check configuration](/consul/docs/discovery/checks) and -[service configuration](/consul/docs/discovery/services) respectively. Service and check -definitions support being updated during a reload. - ```hcl @@ -66,15 +63,27 @@ telemetry { -# Configuration Key Reference ((#config_key_reference)) +### Time-to-live values + +Consul uses the Go `time` package to parse all time-to-live (TTL) values used in Consul agent configuration files. Specify integer and float values as a string and include one or more of the following units of time: + +- `ns` +- `us` +- `µs` +- `ms` +- `s` +- `m` +- `h` + +Examples: + +- `'300ms'` +- `'1.5h'` +- `'2h45m'` --> **Note:** All the TTL values described below are parsed by Go's `time` package, and have the following -[formatting specification](https://golang.org/pkg/time/#ParseDuration): "A -duration string is a possibly signed sequence of decimal numbers, each with -optional fraction and a unit suffix, such as '300ms', '-1.5h' or '2h45m'. -Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." +Refer to the [formatting specification](https://golang.org/pkg/time/#ParseDuration) for additional information. -## General +## General parameters - `addresses` - This is a nested object that allows setting bind addresses. In Consul 1.0 and later these can be set to a space-separated list @@ -914,10 +923,9 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `agent_master` ((#acl_tokens_agent_master)) **Renamed in Consul 1.11 to [`acl.tokens.agent_recovery`](#acl_tokens_agent_recovery).** - - `config_file_service_registration` ((#acl_tokens_config_file_service_registration)) - The ACL - token this agent uses to register services and checks from [service - definitions](/consul/docs/discovery/services) and [check definitions](/consul/docs/discovery/checks) found - in configuration files or in configuration fragments passed to the agent using the `-hcl` + - `config_file_service_registration` ((#acl_tokens_config_file_service_registration)) - Specifies the ACL + token the agent uses to register services and checks from [service](/consul/docs/services/usage/define-services) and [check](/consul/docs/usage/checks) definitions + specified in configuration files or fragments passed to the agent using the `-hcl` flag. If the `token` field is defined in the service or check definition, then that token is used to @@ -1367,7 +1375,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." of `1ns` instead of 0. - `prefer_namespace` ((#dns_prefer_namespace)) **Deprecated in Consul 1.11. - Use the [canonical DNS format for enterprise service lookups](/consul/docs/discovery/dns#service-lookups-for-consul-enterprise) instead.** - + Use the [canonical DNS format for enterprise service lookups](/consul/docs/services/discovery/dns-static-lookups#service-lookups-for-consul-enterprise) instead.** - When set to `true`, in a DNS query for a service, a single label between the domain and the `service` label is treated as a namespace name instead of a datacenter. When set to `false`, the default, the behavior is the same as non-Enterprise diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index 380602081a91..d1860b10a508 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -319,7 +319,7 @@ tls { ### Client node registering a service Using Consul as a central service registry is a common use case. -The following example configuration includes common settings to register a service with a Consul agent and enable health checks (see [Checks](/consul/docs/discovery/checks) to learn more about health checks): +The following example configuration includes common settings to register a service with a Consul agent and enable health checks. Refer to [Define Health Checks](/consul/docs/services/usage/checks) to learn more about health checks. diff --git a/website/content/docs/architecture/anti-entropy.mdx b/website/content/docs/architecture/anti-entropy.mdx index 8c2b9bb96756..39d2a08156cc 100644 --- a/website/content/docs/architecture/anti-entropy.mdx +++ b/website/content/docs/architecture/anti-entropy.mdx @@ -26,7 +26,7 @@ health checks and updating their local state. Services and checks within the context of an agent have a rich set of configuration options available. This is because the agent is responsible for generating information about its services and their health through the use of -[health checks](/consul/docs/discovery/checks). +[health checks](/consul/docs/services/usage/checks). #### Catalog @@ -117,8 +117,7 @@ the source of truth for tag information. For example, the Redis database and its monitoring service Redis Sentinel have this kind of relationship. Redis instances are responsible for much of their configuration, but Sentinels determine whether the Redis instance is a -primary or a secondary. Using the Consul service configuration item -[enable_tag_override](/consul/docs/discovery/services) you can instruct the -Consul agent on which the Redis database is running to NOT update the -tags during anti-entropy synchronization. For more information see -[Services](/consul/docs/discovery/services#enable-tag-override-and-anti-entropy) page. +primary or a secondary. Enable the +[`enable_tag_override`](/consul/docs/services/configuration/services-configuration-reference#enable_tag_override) parameter in your service definition file to tell the Consul agent where the Redis database is running to bypass +tags during anti-entropy synchronization. Refer to +[Modify anti-entropy synchronozation](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional information. diff --git a/website/content/docs/architecture/index.mdx b/website/content/docs/architecture/index.mdx index a36fa644b6b5..a4656a7718b6 100644 --- a/website/content/docs/architecture/index.mdx +++ b/website/content/docs/architecture/index.mdx @@ -55,7 +55,7 @@ You can also run Consul with an alternate service mesh configuration that deploy ## LAN gossip pool -Client and server agents participate in a LAN gossip pool so that they can distribute and perform node [health checks](/consul/docs/discovery/checks). Agents in the pool propagate the health check information across the cluster. Agent gossip communication occurs on port `8301` using UDP. Agent gossip falls back to TCP if UDP is not available. Refer to [Gossip Protocol](/consul/docs/architecture/gossip) for additional information. +Client and server agents participate in a LAN gossip pool so that they can distribute and perform node [health checks](/consul/docs/services/usage/checks). Agents in the pool propagate the health check information across the cluster. Agent gossip communication occurs on port `8301` using UDP. Agent gossip falls back to TCP if UDP is not available. Refer to [Gossip Protocol](/consul/docs/architecture/gossip) for additional information. The following simplified diagram shows the interactions between servers and clients. diff --git a/website/content/docs/architecture/scale.mdx b/website/content/docs/architecture/scale.mdx index 47176c60afd6..ecf5035f98bc 100644 --- a/website/content/docs/architecture/scale.mdx +++ b/website/content/docs/architecture/scale.mdx @@ -236,7 +236,7 @@ Several factors influence Consul performance at scale when used primarily for it - Rate of catalog updates, which is affected by the following events: - A service instance’s health check status changes - A service instance’s node loses connectivity to Consul servers - - The contents of the [service definition file](/consul/docs/discovery/services#service-definition) changes + - The contents of the [service definition file](/consul/docs/services/configuration/services-configuration-reference) changes - Service instances are registered or deregistered - Orchestrators such as Kubernetes or Nomad move a service to a new node @@ -280,4 +280,8 @@ At scale, using Consul as a backend for Vault results in increased memory and CP In situations where Consul handles large amounts of data and has high write throughput, we recommend adding monitoring for the [capacity and health of raft replication on servers](/consul/docs/agent/telemetry#raft-replication-capacity-issues). If the server experiences heavy load when the size of its stored data is large enough, a follower may be unable to catch up on replication and become a voter after restarting. This situation occurs when the time it takes for a server to restore from disk takes longer than it takes for the leader to write a new snapshot and truncate its logs. Refer to [Raft snapshots](#raft-snapshots) for more information. +<<<<<<< HEAD Vault v1.4 and higher provides [integrated storage](/vault/docs/concepts/integrated-storage) as its recommended storage option. If you currently use Consul as a storage backend for Vault, we recommend switching to integrated storage. For a comparison between Vault's integrated storage and Consul as a backend for Vault, refer to [storage backends in the Vault documentation](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). For detailed guidance on migrating the Vault backend from Consul to Vault's integrated storage, refer to the [storage migration tutorial](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). Integrated storage improves resiliency by preventing a Consul outage from also affecting Vault functionality. +======= +Vault v1.4 and higher provides [integrated storage](/vault/docs/concepts/integrated-storage) as its recommended storage option. If you currently use Consul as a storage backend for Vault, we recommend switching to integrated storage. For a comparison between Vault's integrated storage and Consul as a backend for Vault, refer to [storage backends in the Vault documentation](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). For detailed guidance on migrating the Vault backend from Consul to Vault's integrated storage, refer to the [storage migration tutorial](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). Integrated storage improves resiliency by preventing a Consul outage from also affecting Vault functionality. +>>>>>>> main diff --git a/website/content/docs/connect/cluster-peering/tech-specs.mdx b/website/content/docs/connect/cluster-peering/tech-specs.mdx index 3e00d6a48b55..e34c0d190938 100644 --- a/website/content/docs/connect/cluster-peering/tech-specs.mdx +++ b/website/content/docs/connect/cluster-peering/tech-specs.mdx @@ -45,7 +45,7 @@ Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) ## Sidecar proxy requirements -The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/discovery/services). +The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/services/usage/defin-services). - Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams`](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) documentation for details. - The `proxy.upstreams.destination_name` parameter is always required. diff --git a/website/content/docs/connect/config-entries/ingress-gateway.mdx b/website/content/docs/connect/config-entries/ingress-gateway.mdx index ee94412661ca..b1cb5c87ab49 100644 --- a/website/content/docs/connect/config-entries/ingress-gateway.mdx +++ b/website/content/docs/connect/config-entries/ingress-gateway.mdx @@ -172,10 +172,9 @@ gateway: - All services with the same [protocol](/consul/docs/connect/config-entries/ingress-gateway#protocol) as the listener will be routable. -- The ingress gateway will route traffic based on the host/authority header, - expecting a value matching `.ingress.*`, or if using namespaces, - `.ingress..*`. This matches the [Consul DNS - ingress subdomain](/consul/docs/discovery/dns#ingress-service-lookups). +- The ingress gateway routes traffic based on the host or authority header and expects a value matching either `.ingress.*` or + `.ingress..*`. The query matches the [Consul DNS + ingress subdomain](/consul/docs/services/discovery/dns-static-lookups#ingress-service-lookups). A wildcard specifier cannot be set on a listener of protocol `tcp`. diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 9245d8112c91..3dd9812d3c0d 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -1,25 +1,1165 @@ --- layout: docs -page_title: Service Defaults - Configuration Entry Reference -description: >- - The service defaults configuration entry kind defines sets of default configurations that apply to all services in the mesh. Use the examples learn how to define a default protocol, default upstream configuration, and default terminating gateway. +page_title: Service Defaults Configuration Reference +description: -> + Use the service-defaults configuration entry to set default configurations for services, such as upstreams, protocols, and namespaces. Learn how to configure service-defaults. --- -# Service Defaults Configuration Entry -The `service-defaults` config entry kind (`ServiceDefaults` on Kubernetes) controls default global values for a -service, such as its protocol. +# Service Defaults Configuration Reference +This topic describes how to configure service defaults configuration entries. The service defaults configuration entry contains common configuration settings for service mesh services, such as upstreams and gateways. Refer to [Define service defaults](/consul/docs/services/usage/define-services#define-service-defaults) for usage information. -## Sample Config Entries +## Configuration model -### Default protocol +The following outline shows how to format the service splitter configuration entry. Click on a property name to view details about the configuration. --> **NOTE**: The default protocol can also be configured globally for all proxies -using the [proxy defaults](/consul/docs/connect/config-entries/proxy-defaults#default-protocol) -config entry. However, if the protocol value is specified in a service defaults -config entry for a given service, that value will take precedence over the -globally configured value from proxy defaults. + + + +- [`Kind`](#kind): string | required +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string +- [`Partition`](#partition): string +- [`Meta`](#meta): map | no default +- [`Protocol`](#protocol): string | default: `tcp` +- [`BalanceInboundConnections`](#balanceinboundconnections): string | no default +- [`Mode`](#mode): string | no default +- [`UpstreamConfig`](#upstreamconfig): map | no default + - [`Overrides`](#upstreamconfig-overrides): map | no default + - [`Name`](#upstreamconfig-overrides-name): string | no default + - [`Namespace`](#upstreamconfig-overrides-namespace): string | no default + - [`Protocol`](#upstreamconfig-overrides-protocol): string | no default + - [`ConnectTimeoutMs`](#upstreamconfig-overrides-connecttimeoutms): int | default: `5000` + - [`MeshGateway`](#upstreamconfig-overrides-meshgateway): map | no default + - [`mode`](#upstreamconfig-overrides-meshgateway): string | no default + - [`BalanceOutboundConnections`](#upstreamconfig-overrides-balanceoutboundconnections): string | no default + - [`Limits`](#upstreamconfig-overrides-limits): map | optional + - [`MaxConnections`](#upstreamconfig-overrides-limits): integer | `0` + - [`MaxPendingRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`MaxConcurrentRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`PassiveHealthCheck`](#upstreamconfig-overrides-passivehealthcheck): map | optional + - [`Interval`](#upstreamconfig-overrides-passivehealthcheck): string | `0s` + - [`MaxFailures`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`EnforcingConsecutive5xx`](#upstreamconfig-overrides-passivehealthcheck): integer | `100` + - [`Defaults`](#upstreamconfig-defaults): map | no default + - [`Protocol`](#upstreamconfig-defaults-protocol): string | no default + - [`ConnectTimeoutMs`](#upstreamconfig-defaults-connecttimeoutms): int | default: `5000` + - [`MeshGateway`](#upstreamconfig-defaults-meshgateway): map | no default + - [`mode`](#upstreamconfig-defaults-meshgateway): string | no default + - [`BalanceOutboundConnections`](#upstreamconfig-defaults-balanceoutboundconnections): string | no default + - [`Limits`](#upstreamconfig-defaults-limits): map | optional + - [`MaxConnections`](#upstreamconfig-defaults-limits): integer | `0` + - [`MaxPendingRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`MaxConcurrentRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`PassiveHealthCheck`](#upstreamconfig-defaults-passivehealthcheck): map | optional + - [`Interval`](#upstreamconfig-defaults-passivehealthcheck): string | `0s` + - [`MaxFailures`](#upstreamconfig-defaults-passivehealthcheck): integer | `0` + - [`EnforcingConsecutive5xx`](#upstreamconfig-defaults-passivehealthcheck): integer | +- [`TransparentProxy`](#transparentproxy): map | no default + - [`OutboundListenerPort`](#transparentproxy): integer | `15001` + - [`DialedDirectly`](#transparentproxy ): boolean | `false` +- [`Destination`](#destination): map | no default + - [`Addresses`](#destination): list | no default + - [`Port`](#destination): integer | `0` +- [`MaxInboundConnections`](#maxinboundconnections): integer | `0` +- [`LocalConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` +- [`LocalRequestTiimeoutMs`](#localrequesttimeoutms): integer | `0` +- [`MeshGateway`](#meshgateway): map | no default + - [`Mode`](#meshgateway): string | no default +- [`ExternalSNI`](#externalsni): string | no default +- [`Expose`](#expose): map | no default + - [`Checks`](#expose-checks): boolean | `false` + - [`Paths`](#expose-paths): list | no default + - [`Path`](#expose-paths): string | no default + - [`LocalPathPort`](#expose-paths): integer | `0` + - [`ListenerPort`](#expose-paths): integer | `0` + - [`Protocol`](#expose-paths): string | `http` + + + + +- [`apiVersion`](#apiversion): string | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | no default +- [`metadata`](#metadata): map | no default + - [`name`](#name): string | no default + - [`namespace`](#namespace): string | no default | + - [`partition`](#partition): string | no default | +- [`spec`](#spec): map | no default + - [`protocol`](#protocol): string | default: `tcp` + - [`balanceInboundConnections`](#balanceinboundconnections): string | no default + - [`mode`](#mode): string | no default + - [`upstreamConfig`](#upstreamconfig): map | no default + - [`overrides`](#upstreamconfig-overrides): list | no default + - [`name`](#upstreamconfig-overrides-name): string | no default + - [`namespace`](#upstreamconfig-overrides-namespace): string | no default + - [`protocol`](#upstreamconfig-overrides-protocol): string | no default + - [`connectTimeoutMs`](#upstreamconfig-overrides-connecttimeoutms): int | default: `5000` + - [`meshGateway`](#upstreamconfig-overrides-meshgateway): map | no default + - [`mode`](#upstreamconfig-overrides-meshgateway): string | no default + - [`balanceOutboundConnections`](#overrides-balanceoutboundconnections): string | no default + - [`limits`](#upstreamconfig-overrides-limits): map | optional + - [`maxConnections`](#upstreamconfig-overrides-limits): integer | `0` + - [`maxPendingRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`maxConcurrentRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`passiveHealthCheck`](#upstreamconfig-overrides-passivehealthcheck): map | optional + - [`interval`](#upstreamconfig-overrides-passivehealthcheck): string | `0s` + - [`maxFailures`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`mnforcingConsecutive5xx`](#upstreamconfig-overrides-passivehealthcheck): integer | `100` + - [`defaults`](#upstreamconfig-defaults): map | no default + - [`protocol`](#upstreamconfig-defaults-protocol): string | no default + - [`connectTimeoutMs`](#upstreamconfig-defaults-connecttimeoutms): int | default: `5000` + - [`meshGateway`](#upstreamconfig-defaults-meshgateway): map | no default + - [`mode`](#upstreamconfig-defaults-meshgateway): string | no default + - [`balanceOutboundConnections`](#upstreamconfig-defaults-balanceoutboundconnections): string | no default + - [`limits`](#upstreamconfig-defaults-limits): map | optional + - [`maxConnections`](#upstreamconfig-defaults-limits): integer | `0` + - [`maxPendingRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`maxConcurrentRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`passiveHealthCheck`](#upstreamconfig-defaults-passivehealthcheck): map | optional + - [`interval`](#upstreamconfig-defaults-passivehealthcheck): string | `0s` + - [`maxFailures`](#upstreamconfig-defaults-passivehealthcheck): integer | `0` + - [`enforcingConsecutive5xx`](#upstreamconfig-defaults-passivehealthcheck): integer | + - [`transparentProxy`](#transparentproxy): map | no default + - [`outboundListenerPort`](#transparentproxy): integer | `15001` + - [`dialedDirectly`](#transparentproxy): boolean | `false` + - [`destination`](#destination): map | no default + - [`addresses`](#destination): list | no default + - [`port`](#destination): integer | `0` + - [`maxInboundConnections`](#maxinboundconnections): integer | `0` + - [`localConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` + - [`localRequestTiimeoutMs`](#localrequesttimeoutms): integer | `0` + - [`meshGateway`](#meshgateway): map | no default + - [`mode`](#meshgateway): string | no default + - [`externalSNI`](#externalsni): string | no defaiult + - [`expose`](#expose): map | no default + - [`checks`](#expose-checks): boolean | `false` + - [`paths`](#expose-paths): list | no default + - [`path`](#expose-paths): string | no default + - [`localPathPort`](#expose-paths): integer | `0` + - [`listenerPort`](#expose-paths): integer | `0` + - [`protocol`](#expose-paths): string | `http` + + + + +## Complete configuration + +When every field is defined, a service splitter configuration entry has the following form: + + + + +```hcl +Kind = "service-defaults" +Name = "service_name" +Namespace = "namespace" +Partition = "partition" +Meta = { + Key = "value" +} +Protocol = "tcp" +BalanceInboundConnections = "exact_balance" +Mode = "transparent" +UpstreamConfig = { + Overrides = { + Name = "name-of-upstreams-to-override" + Namespace = "namespace-containing-upstreams-to-override" + Protocol = "http" + ConnectTimeoutMs = 100 + MeshGateway = { + mode = "remote" + } + BalanceOutboundConnections = "exact_balance" + Limits = { + MaxConnections = 10 + MaxPendingRequests = 50 + MaxConcurrentRequests = 100 + } + PassiveHealthCheck = { + Interval = "5s" + MaxFailures = 5 + EnforcingConsecutive5xx = 99 + } + } + Defaults = { + Protocol = "http2" + ConnectTimeoutMs = 2000 + MeshGateway = { + mode = "local" + } + BalanceOutboundConnections = "exact_balance" + Limits = { + MaxConnections = 100 + MaxPendingRequests = 500 + MaxConcurrentRequests = 1000 + } + PassiveHealthCheck = { + Interval = "1s" + MaxFailures = 1 + EnforcingConsecutive5xx = 89 + } + } +} +TransparentProxy = { + OutboundListenerPort = 15002 + DialedDirectly = true +} +Destination = { + Addresses = [ + "First IP address", + "Second IP address" + ] + Port = 88 +} +MaxInboundConnections = 100 +LocalConnectTimeoutMs = 10 +LocalRequestTimeoutMs = 10 +MeshGateway = { + Mode = "remote" +} +ExternalSNI = "sni-server-host" +Expose = { + Checks = true + Paths = [ + { + Path = "/local/dir" + LocalPathPort = 99 + LocalListenerPort = 98 + Protocol = "http2" + } + ] +} +``` + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: + namespace: + partition: +spec: + protocol: tcp + balanceInboundConnnections: exact_balance + mode: transparent + upstreamConfig: + overrides: + - name: + namespace: + protocol: + connectTimeoutMs: 5000 + meshGateway: + mode: + balanceOutboundConnections: exact_balance + limits: + maxConnections: 0 + maxPendingRequests: 0 + maxConcurrentRequests: 0 + passiveHealthCheck: + interval: 0s + maxFailures: 0 + enforcingConsecutive5xx: 100 + defaults: + protocol: + connectTimeoutMs: 5000 + meshGateway: + mode: + balanceOutboundConnections: exact_balance + limits: + maxConnections: 0 + maxPendingRequests: 0 + maxConcurrentRequests: 0 + passiveHealthCheck: + interval: 0s + maxFailures: 0 + enforcingConsecutive5xx: 100 + transparentProxy: + outboundListenerPort: 15001 + dialedDirectly: false + destination: + addresses: + - + + port: 0 + maxInboundConnections: 0 + meshGateway: + mode: + externalSNI: + expose: + checks: false + paths: + - path: + localPathPort: 0 + listenerPort: 0 + protocol: http +``` + + + + + +```json +{ + "apiVersion": "consul.hashicorp.com/v1alpha1", + "kind": "ServiceDefaults", + "metadata": { + "name": "", + "namespace": "", + "partition": "" + }, + "spec": { + "protocol": "tcp", + "balanceInboundConnnections": "exact_balance", + "mode": "transparent", + "upstreamConfig": { + "overrides": [ + { + "name": "", + "namespace": "", + "protocol": "", + "connectTimeoutMs": 5000, + "meshGateway": { + "mode": "" + }, + "balanceOutboundConnections": "exact_balance", + "limits": { + "maxConnections": 0, + "maxPendingRequests": 0, + "maxConcurrentRequests": 0 + }, + "passiveHealthCheck": { + "interval": "0s", + "maxFailures": 0, + "enforcingConsecutive5xx": 100 + } + } + ], + "defaults": { + "protocol": "", + "connectTimeoutMs": 5000, + "meshGateway": { + "mode": "" + }, + "balanceOutboundConnections": "exact_balance", + "limits": { + "maxConnections": 0, + "maxPendingRequests": 0, + "maxConcurrentRequests": 0 + }, + "passiveHealthCheck": { + "interval": "0s", + "maxFailures": 0, + "enforcingConsecutive5xx": 100 + } + } + }, + "transparentProxy": { + "outboundListenerPort": 15001, + "dialedDirectly": false + }, + "destination": { + "addresses": [ + "", + "" + ], + "port": 0 + }, + "maxInboundConnections": 0, + "meshGateway": { + "mode": "" + }, + "externalSNI": "", + "expose": { + "checks": false, + "paths": [ + { + "path": "", + "localPathPort": 0, + "listenerPort": 0, + "protocol": "http" + } + ] + } + } +} +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service defaults configuration entry. + + + + +### `Kind` + +Specifies the configuration entry type. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `service-defaults`. + +### `Name` + +Specifies the name of the service you are setting the defaults for. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Consul namespace that the configuration entry applies to. + +#### Values + +- Default: `default` +- Data type: string + +### `Partition` + +Specifies the name of the name of the Consul admin partition that the configuration entry applies to. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information. + +#### Values + +- Default: `default` +- Data type: string + +### `Meta` + +Specifies a set of custom key-value pairs to add to the [Consul KV](/consul/docs/dynamic-app-config/kv) store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs. + - keys: string + - values: string, integer, or float + +### `Protocol` + +Specifies the default protocol for the service. In service mesh use cases, the `protocol` configuration is required to enable the following features and components: + +- [observability](/consul/docs/connect/observability) +- [service splitter configuration entry](/consul/docs/connect/config-entries/service-splitter) +- [service router configuration entry](/consul/docs/connect/config-entries/service-router) +- [L7 intentions](/consul/docs/connect/intentions) + +You can set the global protocol for proxies in the [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#default-protocol) configuration entry, but the protocol specified in the `service-defaults` configuration entry overrides the `proxy-defaults` configuration. + +#### Values + +- Default: `tcp` +- You can speciyf one of the following string values: + - `tcp` (default) + - `http` + - `http2` + - `grpc` + +Refer to [Set the default protocol](#set-the-default-protocol) for an example configuration. + +### `BalanceInboundConnections` + +Specifies the strategy for allocating inbound connections to the service across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `Mode` + +Specifies a mode for how the service directs inbound and outbound traffic. + +- Default: none +- You can specify the following string values: + - `direct`: The proxy's listeners must be dialed directly by the local application and other proxies. + - `transparent`: The service captures inbound and outbound traffic and redirects it through the proxy. The mode does not enable the traffic redirection. It instructs Consul to configure Envoy as if traffic is already being redirected. + + +### `UpstreamConfig` + +Controls default upstream connection settings and custom overrides for individual upstream services. If your network contains federated datacenters, individual upstream configurations apply to all pairs of source and upstream destination services in the network. Refer to the following fields for details: + +- [`UpstreamConfig.Overrides`](#upstreamconfig-overrides) +- [`UpstreamConfig.Defaults`](#upstreamconfig-defaults) + +#### Values + +- Default: none +- Data type: map + +### `UpstreamConfig.Overrides[]` + +Specifies options that override the [default upstream configurations](#upstreamconfig-defaults) for individual upstreams. + +#### Values + +- Default: none +- Data type: list + +### `UpstreamConfig.Overrides[].Name` + +Specifies the name of the upstream service that the configuration applies to. We recommend that you do not use the `*` wildcard to avoid applying the configuration to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Namespace` + +Specifies the namespace containing the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from appling to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Protocol` +Specifies the protocol to use for requests to the upstream listener. + +We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: none +- Data type: string + + +### `UpstreamConfig.Overrides[].ConnectTimeoutMs` + +Specifies how long in milliseconds that the service should attempt to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connection_timeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `service-resolver` configuration entry for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `service-defaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `UpstreamConfig.Overrides[].MeshGateway` + +Map that contains the default mesh gateway `mode` field for the upstream. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +- Default: `none` +- You can specify the following string values for the `mode` field: + - `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. + - `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. + - `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + + +### `UpstreamConfig.Overrides[].BalanceOutboundConnections` + +Sets the strategy for allocating outbound connections from the upstream across Envoy proxy threads. + +#### Values + +The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Limits` + +Map that specifies a set of limits to apply to when connecting to individual upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `MaxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `MaxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `MaxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +Refer to the [upstream configuration example](#upstream-configuration) for additional guidance. + +### `UpstreamConfig.Overrides[].PassiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes passive health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `Interval` | Specifies the time between checks. | string | `0s` | +| `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `EnforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + +### `UpstreamConfig.Defaults` + +Specifies configurations that set default upstream settings. For information about overriding the default configurations for in for individual upstreams, refer to [`UpstreamConfig.Overrides`](#upstreamconfig-overrides). + +#### Values + +- Default: none +- Data type: map + +### `UpstreamConfig.Defaults.Protocol` + +Specifies default protocol for upstream listeners. + +We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +- Default: none +- Data type: string + +### `UpstreamConfig.Defaults.ConnectTimeoutMs` + +Specifies how long in milliseconds that all services should continue attempting to establish an upstream connection before timing out. + +For non-Kubernetes environments, we recommend configuring the upstream timeout in the [`connection_timeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `service-resolver` configuration entry for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `service-defaults` upstream configuration limits L7 management functionality. + +- Default: `5000` +- Data type: integer + +### `UpstreamConfig.Defaults.MeshGateway` + +Specifies the default mesh gateway `mode` field for all upstreams. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `UpstreamConfig.Defaults.BalanceOutboundConnections` + +Sets the strategy for allocating outbound connections from upstreams across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +- Default: none +- Data type: string + +### `UpstreamConfig.Defaults.Limits` + +Map that specifies a set of limits to apply to when connecting upstream services. The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `MaxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `MaxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `MaxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `UpstreamConfig.Defaults.PassiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. The following table describes the health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `Interval` | Specifies the time between checks. | string | `0s` | +| `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `EnforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + +### `TransparentProxy` + +Controls configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. + +You can configure the following parameters in the `TransparentProxy` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `OutboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | +| `DialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | + +### `Destination[]` -Set the default protocol for a service in the default namespace to HTTP: +Configures the destination for service traffic through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. + +You can configure the following parameters in the `Destination` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Address` | Specifies a list of addresses for the destination. You can configure a list of hostnames and IP addresses. Wildcards are not supported. | list | none | +| `Port` | Specifies the port number of the destination. | integer | `0` | + +### `MaxInboundConnections` + +Specifies the maximum number of concurrent inbound connections to each service instance. + +- Default: `0` +- Data type: integer + +### `LocalConnectTimeoutMs` + +Specifies the number of milliseconds allowed for establishing connections to the local application instance before timing out. + +- Default: `5000` +- Data type: integer + +### `LocalRequestTimeoutMs` + +Specifies the timeout for HTTP requests to the local application instance. Applies to HTTP-based protocols only. If not specified, inherits the Envoy default for route timeouts. + +- Default: Inherits `15s` from Envoy as the default +- Data type: string + +### `MeshGateway` + +Specifies the default mesh gateway `mode` field for the service. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `ExternalSNI` + +Specifies the TLS server name indication (SNI) when federating with an external system. + +- Default: none +- Data type: string + +### `Expose` + +Specifies default configurations for exposing HTTP paths through Envoy. Exposing paths through Envoy enables services to listen on localhost only. Applications that are not Consul service mesh-enabled can still contact an HTTP endpoint. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information and example configurations. + +- Default: none +- Data type: map + +### `Expose.Checks` + +Exposes all HTTP and gRPC checks registered with the agent if set to `true`. Envoy exposes listeners for the checks and only accepts connections originating from localhost or Consul's [`advertise_addr`](/consul/docs/agent/config/config-files#advertise_addr). The ports for the listeners are dynamically allocated from the agent's [`expose_min_port`](/consul/docs/agent/config/config-files#expose_min_port) and [`expose_max_port`](/consul/docs/agent/config/config-files#expose_max_port) configurations. + +We recommend enabling the `Checks` configuration when a Consul client cannot reach registered services over localhost, such as when Consul agents run in their own pods in Kubernetes. + +- Default: `false` +- Data type: boolean + +### `Expose.Paths[]` + +Specifies a list of configuration maps that define paths to expose through Envoy when `Expose.Checks` is set to `true`. You can configure the following parameters for each map in the list: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Path` | Specifies the HTTP path to expose. You must prepend the path with a forward slash (`/`). | string | none | +| `LocalPathPort` | Specifies the port where the local service listens for connections to the path. | integer | `0` | +| `ListenPort` | Specifies the port where the proxy listens for connections. The port must be available. If the port is unavailable, Envoy does not expose a listener for the path and the proxy registration still succeeds. | integer | `0` | +| `Protocol` | Specifies the protocol of the listener. You can configure one of the following values:
  • `http`
  • `http2`: Use with gRPC traffic
    • | integer | `http` | + +     + + + +### `apiVersion` + +Specifies the version of the Consul API for integrating with Kubernetes. The value must be `consul.hashicorp.com/v1alpha1`. The `apiVersion` field is not supported for non-Kubernetes deployments. + +- Default: none +- This field is required. +- String value that must be set to `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the configuration entry type. Must be ` ServiceDefaults`. + +- Required: required +- String value that must be set to `ServiceDefaults`. + +### `metadata` + +Map that contains the service name, namespace, and admin partition that the configuration entry applies to. + +#### Values + +- Default: none +- Map containing the following strings: + - [`name`](#name) + - [`namespace`](#namespace) + - [`partition`](#partition) + + +### `metadata.name` + +Specifies the name of the service you are setting the defaults for. + +#### Values + +- Default: none +- This field is required +- Data type: string + +### `metadata.namespace` + +Specifies the Consul namespace that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +- Default: `default` +- Data type: string + +### `metadata.partition` + +Specifies the name of the name of the Consul admin partition that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul Enterprise on Kubernetes. Consul OSS distributions ignore the `metadata.partition` configuration. + +- Default: `default` +- Data type: string + +### `spec` + +Map that contains the details about the `ServiceDefaults` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the `spec` field. All other configurations are children. + +### `spec.protocol` + +Specifies the default protocol for the service. In service service mesh use cases, the `protocol` configuration is required to enable the following features and components: + +- [observability](/consul/docs/connect/observability) +- [`service-splitter` configuration entry](/consul/docs/connect/config-entries/service-splitter) +- [`service-router` configuration entry](/consul/docs/connect/config-entries/service-router) +- [L7 intentions](/consul/docs/connect/intentions) + +You can set the global protocol for proxies in the [`ProxyDefaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults#default-protocol), but the protocol specified in the `ServiceDefaults` configuration entry overrides the `ProxyDefaults` configuration. + +#### Values + +- Default: `tcp` +- You can specify one of the following string values: + - `tcp` + - `http` + - `http2` + - `grpc` + +Refer to [Set the default protocol](#set-the-default-protocol) for an example configuration. + +### `spec.balanceInboundConnections` + +Specifies the strategy for allocating inbound connections to the service across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.mode` + +Specifies a mode for how the service directs inbound and outbound traffic. + +#### Values + +- Default: none +- Required: optional +- You can specified the following string values: + +- `direct`: The proxy's listeners must be dialed directly by the local application and other proxies. +- `transparent`: The service captures inbound and outbound traffic and redirects it through the proxy. The mode does not enable the traffic redirection. It instructs Consul to configure Envoy as if traffic is already being redirected. + +### `spec.upstreamConfig` + +Specifies a map that controls default upstream connection settings and custom overrides for individual upstream services. If your network contains federated datacenters, individual upstream configurations apply to all pairs of source and upstream destination services in the network. + +#### Values + +- Default: none +- Map that contains the following configurations: + - [`UpstreamConfig.Overrides`](#upstreamconfig-overrides) + - [`UpstreamConfig.Defaults`](#upstreamconfig-defaults) + +### `spec.upstreamConfig.overrides[]` + +Specifies options that override the [default upstream configurations](#spec-upstreamconfig-defaults) for individual upstreams. + +#### Values + +- Default: none +- Data type: list + +### `spec.upstreamConfig.overrides[].name` + +Specifies the name of the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from applying to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].namespace` + +Specifies the namespace containing the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from applying to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].protocol` + +Specifies the protocol to use for requests to the upstream listener. We recommend configuring the protocol in the main [`protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: inherits the main [`protocol`](#protocol) configuration +- Data type: string + + +### `spec.upstreamConfig.overrides[].connectTimeoutMs` + +Specifies how long in milliseconds that the service should attempt to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connectTimeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `ServiceResolver` CRD for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `ServiceDefaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.upstreamConfig.overrides[].meshGateway.mode` + +Map that contains the default mesh gateway `mode` field for the upstream. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.upstreamConfig.overrides[].balanceInboundConnections` + +Sets the strategy for allocating outbound connections from the upstream across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].limits` + +Map that specifies a set of limits to apply to when connecting to individual upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `maxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `maxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `maxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `spec.upstreamConfig.overrides[].passiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes passive health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `interval` | Specifies the time between checks. | string | `0s` | +| `maxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `enforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + +### `spec.upstreamConfig.defaults` + +Map of configurations that set default upstream configurations for the service. For information about overriding the default configurations for in for individual upstreams, refer to [`spec.upstreamConfig.overrides`](#spec-upstreamconfig-overrides). + +#### Values + +- Default: none +- Data type: list + +### `spec.upstreamConfig.defaults.protocol` + +Specifies default protocol for upstream listeners. We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.default.connectTimeoutMs` + +Specifies how long in milliseconds that all services should continue attempting to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connectTimeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `ServiceResolver` CRD for upstream destination services. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `ServiceDefaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.upstreamConfig.defaults.meshGateway.mode` + +Specifies the default mesh gateway `mode` field for all upstreams. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.upstreamConfig.defaults.balanceInboundConnections` + +Sets the strategy for allocating outbound connections from upstreams across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.defaults.limits` + +Map that specifies a set of limits to apply to when connecting upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `maxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `maxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `maxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `spec.upstreamConfig.defaults.passiveHealthCheck` +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes the health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `interval` | Specifies the time between checks. | string | `0s` | +| `maxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `enforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + +### `spec.transparentProxy` + +Map of configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. + +#### Values + +You can configure the following parameters in the `TransparentProxy` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `outboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | +| `dialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | + +### `spec.destination` + +Map of configurations that specify one or more destinations for service traffic routed through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. + +#### Values + +You can configure the following parameters in the `Destination` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `address` | Specifies a list of addresses for the destination. You can configure a list of hostnames and IP addresses. Wildcards are not supported. | list | none | +| `port` | Specifies the port number of the destination. | integer | `0` | + +### `spec.maxInboundConnections` + +Specifies the maximum number of concurrent inbound connections to each service instance. + +#### Values + +- Default: `0` +- Data type: integer + +### `spec.localConnectTimeoutMs` + +Specifies the number of milliseconds allowed for establishing connections to the local application instance before timing out. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.localRequestTimeoutMs` + +Specifies the timeout for HTTP requests to the local application instance. Applies to HTTP-based protocols only. If not specified, inherits the Envoy default for route timeouts. + +#### Values + +- Default of `15s` is inherited from Envoy +- Data type: string + +### `spec.meshGateway.mode` +Specifies the default mesh gateway `mode` field for the service. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.externalSNI` + +Specifies the TLS server name indication (SNI) when federating with an external system. + +#### Values + +- Default: none +- Data type: string + +### `spec.expose` + +Specifies default configurations for exposing HTTP paths through Envoy. Exposing paths through Envoy enables services to listen on localhost only. Applications that are not Consul service mesh-enabled can still contact an HTTP endpoint. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information and example configurations. + +#### Values + +- Default: none +- Data type: string + +### `spec.expose.checks` + +Exposes all HTTP and gRPC checks registered with the agent if set to `true`. Envoy exposes listeners for the checks and only accepts connections originating from localhost or Consul's [`advertise_addr`](/consul/docs/agent/config/config-files#advertise_addr). The ports for the listeners are dynamically allocated from the agent's [`expose_min_port`](/consul/docs/agent/config/config-files#expose_min_port) and [`expose_max_port`](/consul/docs/agent/config/config-files#expose_max_port) configurations. + +We recommend enabling the `Checks` configuration when a Consul client cannot reach registered services over localhost, such as when Consul agents run in their own pods in Kubernetes. + +#### Values + +- Default: `false` +- Data type: boolean + +### `spec.expose.paths[]` + +Specifies an list of maps that define paths to expose through Envoy when `spec.expose.checks` is set to `true`. + +#### Values + +The following table describes the parameters for each map: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `path` | Specifies the HTTP path to expose. You must prepend the path with a forward slash (`/`). | string | none | +| `localPathPort` | Specifies the port where the local service listens for connections to the path. | integer | `0` | +| `listenPort` | Specifies the port where the proxy listens for connections. The port must be available. If the port is unavailable, Envoy does not expose a listener for the path and the proxy registration still succeeds. | integer | `0` | +| `protocol` | Specifies the protocol of the listener. You can configure one of the following values:
  • `http`
  • `http2`: Use with gRPC traffic
    • | integer | `http` | + +     +     + +## Example configurations + +The following examples describe common `service-defaults` configurations. + +### Set the default protocol + +In the following example, protocol for the `web` service in the `default` namespace is set to `http`: @@ -50,14 +1190,15 @@ spec: +You can also set the global default protocol for all proxies in the [`proxy-defaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults#default-protocol), but the protocol specified for individual service instances in the `service-defaults` configuration entry takes precedence over the globally-configured value set in the `proxy-defaults`. + ### Upstream configuration -Set default connection limits and mesh gateway mode across all upstreams -of "dashboard", and also override the mesh gateway mode used when dialing -its upstream "counting" service. +The following example sets default connection limits and mesh gateway mode across all upstreams of the `dashboard` service. +It also overrides the mesh gateway mode used when dialing its `counting` upstream service. @@ -140,10 +1281,7 @@ spec: -Set default connection limits and mesh gateway mode across all upstreams -of "dashboard" in the "product" namespace, -and also override the mesh gateway mode used when dialing -its upstream "counting" service in the "backend" namespace. +The following example configures the default connection limits and mesh gateway mode for all of the `counting` service's upstreams. It also overrides the mesh gateway mode used when dialing the `dashboard` service in the `frontend` namespace. @@ -234,8 +1372,8 @@ spec: ### Terminating gateway destination -Create a default destination that will be assigned to a terminating gateway. A destination -represents a location outside the Consul cluster. They can be dialed directly when transparent proxy mode is enabled. +The following examples creates a default destination assigned to a terminating gateway. A destination +represents a location outside the Consul cluster. Services can dial destinations dialed directly when transparent proxy mode is enabled. @@ -276,6 +1414,7 @@ represents a location outside the Consul cluster. They can be dialed directly wh + \ No newline at end of file diff --git a/website/content/docs/connect/configuration.mdx b/website/content/docs/connect/configuration.mdx index 8fbd88fa3252..e33694688598 100644 --- a/website/content/docs/connect/configuration.mdx +++ b/website/content/docs/connect/configuration.mdx @@ -9,14 +9,11 @@ description: >- There are many configuration options exposed for Consul service mesh. The only option that must be set is the `connect.enabled` option on Consul servers to enable Consul service mesh. -All other configurations are optional and have reasonable defaults. +All other configurations are optional and have defaults suitable for many environments. -Consul Connect is the component shipped with Consul that enables service mesh functionality. The terms _Consul Connect_ and _Consul service mesh_ are used interchangeably throughout this documentation. +The terms _Consul Connect_ and _Consul service mesh_ are used interchangeably throughout this documentation. --> **Tip:** Service mesh is enabled by default when running Consul in -dev mode with `consul agent -dev`. - -## Agent Configuration +## Agent configuration Begin by enabling Connect for your Consul cluster. By default, Connect is disabled. Enabling Connect requires changing @@ -75,18 +72,12 @@ automatically ensure complete security. Please read the [Connect production tutorial](/consul/tutorials/developer-mesh/service-mesh-production-checklist) to understand the additional steps needed for a secure deployment. -## Centralized Proxy and Service Configuration - -To account for common Connect use cases where you have many instances of the -same service, and many colocated sidecar proxies, Consul allows you to customize -the settings for all of your proxies or all the instances of a given service at -once using [Configuration Entries](/consul/docs/agent/config-entries). +## Centralized proxy and service configuration -You can override centralized configurations for individual proxy instances in -their +If your network contains many instances of the same service and many colocated sidecar proxies, you can specify global settings for proxies or services in [Configuration Entries](/consul/docs/agent/config-entries). You can override the centralized configurations for individual proxy instances in their [sidecar service definitions](/consul/docs/connect/registration/sidecar-service), and the default protocols for service instances in their [service -registrations](/consul/docs/discovery/services). +definitions](/consul/docs/services/usage/define-services). ## Schedulers diff --git a/website/content/docs/connect/gateways/ingress-gateway.mdx b/website/content/docs/connect/gateways/ingress-gateway.mdx index d75dbd5ebc06..2ba36b3d79bd 100644 --- a/website/content/docs/connect/gateways/ingress-gateway.mdx +++ b/website/content/docs/connect/gateways/ingress-gateway.mdx @@ -20,7 +20,7 @@ to a set of backing [services](/consul/docs/connect/config-entries/ingress-gateway#services). To enable easier service discovery, a new Consul [DNS -subdomain](/consul/docs/discovery/dns#ingress-service-lookups) is provided, on +subdomain](/consul/docs/services/discovery/dns-static-lookups#ingress-service-lookups) is provided, on `.ingress.`. For listeners with a @@ -29,7 +29,7 @@ For listeners with a case, the ingress gateway relies on host/authority headers to decide the service that should receive the traffic. The host used to match traffic defaults to the [Consul DNS ingress -subdomain](/consul/docs/discovery/dns#ingress-service-lookups), but can be changed using +subdomain](/consul/docs/services/discovery/dns-static-lookups#ingress-service-lookups), but can be changed using the [hosts](/consul/docs/connect/config-entries/ingress-gateway#hosts) field. ![Ingress Gateway Architecture](/img/ingress-gateways.png) diff --git a/website/content/docs/connect/native/index.mdx b/website/content/docs/connect/native/index.mdx index ffeb720f8bf3..6dee2160465a 100644 --- a/website/content/docs/connect/native/index.mdx +++ b/website/content/docs/connect/native/index.mdx @@ -54,7 +54,7 @@ Details on the steps are below: - **Service discovery** - This is normal service discovery using Consul, a static IP, or any other mechanism. If you're using Consul DNS, the - [`.connect`](/consul/docs/discovery/dns#connect-capable-service-lookups) + [`.connect`](/consul/docs/services/discovery/dns-static-lookups#service-mesh-enabled-service-lookups) syntax to find Connect-capable endpoints for a service. After service discovery, choose one address from the list of **service addresses**. @@ -94,7 +94,7 @@ Details on the steps are below: HTTP) aware it can safely enforce intentions per _request_ instead of the coarser per _connection_ model. -## Updating Certificates and Certificate Roots +## Update certificates and certificate roots The leaf certificate and CA roots can be updated at any time and the natively integrated application must react to this relatively quickly @@ -129,14 +129,14 @@ Some language libraries such as the [Go library](/consul/docs/connect/native/go) automatically handle updating and locally caching the certificates. -## Service Registration +## Service registration Connect-native applications must tell Consul that they support Connect natively. This enables the service to be returned as part of service -discovery for Connect-capable services, used by other Connect-native applications +discovery for service mesh-capable services used by other Connect-native applications and client [proxies](/consul/docs/connect/proxies). -This can be specified directly in the [service definition](/consul/docs/discovery/services): +You can enable native service mesh support directly in the [service definition](/consul/docs/services/configuration/services-configuration-reference#connect) by configuring the `connect` block. In the following example, the `redis` service is configured to support service mesh natively: ```json { diff --git a/website/content/docs/connect/observability/index.mdx b/website/content/docs/connect/observability/index.mdx index 3da463b389b3..23dfd81b19ee 100644 --- a/website/content/docs/connect/observability/index.mdx +++ b/website/content/docs/connect/observability/index.mdx @@ -26,7 +26,7 @@ configuration](/consul/docs/agent/config/config-files#enable_central_service_con If you are using Kubernetes, the Helm chart can simplify much of the configuration needed to enable observability. See our [Kubernetes observability docs](/consul/docs/k8s/connect/observability/metrics) for more information. -### Metrics Destination +### Metrics destination For Envoy the metrics destination can be configured in the proxy configuration entry's `config` section. @@ -41,18 +41,18 @@ config { Find other possible metrics syncs in the [Connect Envoy documentation](/consul/docs/connect/proxies/envoy#bootstrap-configuration). -### Service Protocol +### Service protocol -You can specify the [service protocol](/consul/docs/connect/config-entries/service-defaults#protocol) -in the `service-defaults` configuration entry. You can override it in the -[service registration](/consul/docs/discovery/services). By default, proxies only give -you L4 metrics. This protocol allows proxies to handle requests at the right L7 -protocol and emit richer L7 metrics. It also allows proxies to make per-request +You can specify the [`protocol`](/consul/docs/connect/config-entries/service-defaults#protocol) +for all service instances in the `service-defaults` configuration entry. You can also override the default protocol when defining and registering proxies in a service definition file. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information. + +By default, proxies only provide L4 metrics. +Defining the protocol allows proxies to handle requests at the L7 +protocol and emit L7 metrics. It also allows proxies to make per-request load balancing and routing decisions. -### Service Upstreams +### Service upstreams You can set the upstream for each service using the proxy's [`upstreams`](/consul/docs/connect/registration/service-registration#upstreams) -sidecar parameter, which can be defined in a service's [sidecar -registration](/consul/docs/connect/registration/sidecar-service). +sidecar parameter, which can be defined in a service's [sidecar registration](/consul/docs/connect/registration/sidecar-service). diff --git a/website/content/docs/connect/proxies/index.mdx b/website/content/docs/connect/proxies/index.mdx index 799daebd4ed2..f47aad3f5dc7 100644 --- a/website/content/docs/connect/proxies/index.mdx +++ b/website/content/docs/connect/proxies/index.mdx @@ -7,29 +7,23 @@ description: >- # Service Mesh Proxy Overview -A Connect-aware proxy enables unmodified applications to use Connect. A +Proxies enable unmodified applications to connect to other services in the service mesh. A per-service proxy sidecar transparently handles inbound and outbound service connections, automatically wrapping and verifying TLS connections. Consul -includes its own built-in L4 proxy and has first class support for Envoy. You -can choose other proxies to plug in as well. This section describes how to +ships with a built-in L4 proxy and has first class support for Envoy. You +can plug other proxies into your environment as well. This section describes how to configure Envoy or the built-in proxy using Connect, and how to integrate the proxy of your choice. -To ensure that services only allow external connections established via -the Connect protocol, you should configure all services to only accept connections on a loopback address. +To ensure that services only allow external connections established through +the service mesh protocol, you should configure all services to only accept connections on a loopback address. -~> **Deprecation Note:** Managed Proxies are a deprecated method for deploying -sidecar proxies, and have been removed in Consul 1.6. See [managed proxy -deprecation](/consul/docs/connect/proxies/managed-deprecated) for more -information. If you are using managed proxies we strongly recommend that you -switch service definitions for registering proxies. +## Dynamic upstreams require native integration -## Dynamic Upstreams Require Native Integration +Service mesh proxies do not support dynamic upstreams. If an application requires dynamic dependencies that are only available -at runtime, it must [natively integrate](/consul/docs/connect/native) -with Connect. After natively integrating, the HTTP API or -[DNS interface](/consul/docs/discovery/dns#connect-capable-service-lookups) -can be used. - -!> Connect proxies do not currently support dynamic upstreams. +at runtime, you must [natively integrate](/consul/docs/connect/native) +the application with Consul service mesh. After natively integrating, the HTTP API or +[DNS interface](/consul/docs/services/discovery/dns-static-lookups#service-mesh-enabled-service-lookups) +can be used. \ No newline at end of file diff --git a/website/content/docs/connect/proxies/managed-deprecated.mdx b/website/content/docs/connect/proxies/managed-deprecated.mdx deleted file mode 100644 index 4ac7adeb3a82..000000000000 --- a/website/content/docs/connect/proxies/managed-deprecated.mdx +++ /dev/null @@ -1,278 +0,0 @@ ---- -layout: docs -page_title: Managed Proxy for Connect (Legacy) -description: >- - Consul's service mesh originally included a proxy manager that was deprecated in version 1.6. Learn about the reasons for its deprecation and how it worked with this legacy documentation. ---- - -# Managed Proxy for Connect Legacy Documentation - -Consul Connect was first released as a beta feature in Consul 1.2.0. The initial -release included a feature called "Managed Proxies". Managed proxies were -Connect proxies where the proxy process was started, configured, and stopped by -Consul. They were enabled via basic configurations within the service -definition. - -!> **Consul 1.6.0 removes Managed Proxies completely.** -This documentation is provided for prior versions only. You may consider using -[sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) instead. - -Managed proxies have been deprecated since Consul 1.3 and have been fully removed -in Consul 1.6. Anyone using Managed Proxies should aim to change their workflow -as soon as possible to avoid issues with a later upgrade. - -After transitioning away from all managed proxy usage, the `proxy` subdirectory inside [`data_dir`](/consul/docs/agent/config/cli-flags#_data_dir) (specified in Consul config) can be deleted to remove extraneous configuration files and free up disk space. - -**new and known issues will not be fixed**. - -## Deprecation Rationale - -Originally managed proxies traded higher implementation complexity for an easier -"getting started" user experience. After seeing how Connect was investigated and -adopted during beta it became obvious that they were not the best trade off. - -Managed proxies only really helped in local testing or VM-per-service based -models whereas a lot of users jumped straight to containers where they are not -helpful. They also add only targeted fairly basic supervisor features which -meant most people would want to use something else in production for consistency -with other workloads. So the high implementation cost of building robust process -supervision didn't actually benefit most real use-cases. - -Instead of this Connect 1.3.0 introduces the concept of [sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) which -have almost all of the benefits of simpler configuration but without any of the -additional process management complexity. As a result they can be used to -simplify configuration in both container-based and realistic production -supervisor settings. - -## Managed Proxy Documentation - -As the managed proxy features continue to be supported for now, the rest of this -page will document how they work in the interim. - --> **Deprecation Note:** It's _strongly_ recommended you do not build anything -using Managed proxies and consider using [sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) instead. - -Managed proxies are given -a unique proxy-specific ACL token that allows read-only access to Connect -information for the specific service the proxy is representing. This ACL -token is more restrictive than can be currently expressed manually in -an ACL policy. - -The default managed proxy is a basic proxy built-in to Consul and written -in Go. Having a basic built-in proxy allows Consul to have a reasonable default -with performance that is good enough for most workloads. In some basic -benchmarks, the service-to-service communication over the built-in proxy -could sustain 5 Gbps with sub-millisecond latency. Therefore, -the performance impact of even the basic built-in proxy is minimal. - -Consul will be integrating with advanced proxies in the near future to support -more complex configurations and higher performance. The configuration below is -all for the built-in proxy. - --> **Security Note:** 1.) Managed proxies can only be configured -via agent configuration files. They _cannot_ be registered via the HTTP API. -And 2.) Managed proxies are not started at all if Consul is running as root. -Both of these default configurations help prevent arbitrary process -execution or privilege escalation. This behavior can be configured -[per-agent](/consul/docs/agent/config). - -### Lifecycle - -The Consul agent starts managed proxies on demand and supervises them, -restarting them if they crash. The lifecycle of the proxy process is decoupled -from the agent so if the agent crashes or is restarted for an upgrade, the -managed proxy instances will _not_ be stopped. - -Note that this behavior while desirable in production might leave proxy -processes running indefinitely if you manually stop the agent and clear its -data dir during testing. - -To terminate a managed proxy cleanly you need to deregister the service that -requested it. If the agent is already stopped and will not be restarted again, -you may choose to locate the proxy processes and kill them manually. - -While in `-dev` mode, unless a `-data-dir` is explicitly set, managed proxies -switch to being killed when the agent exits since it can't store state in order -to re-adopt them on restart. - -### Minimal Configuration - -Managed proxies are configured within a -[service definition](/consul/docs/discovery/services). The simplest possible -managed proxy configuration is an empty configuration. This enables the -default managed proxy and starts a listener for that service: - -```json -{ - "service": { - "name": "redis", - "port": 6379, - "connect": { "proxy": {} } - } -} -``` - -The listener is started on random port within the configured Connect -port range. It can be discovered using the -[DNS interface](/consul/docs/discovery/dns#connect-capable-service-lookups) -or -[Catalog API](#). -In most cases, service-to-service communication is established by -a proxy configured with upstreams (described below), which handle the -discovery transparently. - -### Upstream Configuration - -To transparently discover and establish Connect-based connections to -dependencies, they must be configured with a static port on the managed -proxy configuration: - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "upstreams": [ - { - "destination_name": "redis", - "local_bind_port": 1234 - } - ] - } - } - } -} -``` - -In the example above, -"redis" is configured as an upstream with static port 1234 for service "web". -When a TCP connection is established on port 1234, the proxy -will find Connect-compatible "redis" services via Consul service discovery -and establish a TLS connection identifying as "web". - -~> **Security:** Any application that can communicate to the configured -static port will be able to masquerade as the source service ("web" in the -example above). You must either trust any loopback access on that port or -use namespacing techniques provided by your operating system. - --> **Deprecation Note:** versions 1.2.0 to 1.3.0 required specifying `upstreams` -as part of the opaque `config` that is passed to the proxy. However, since -1.3.0, the `upstreams` configuration is now specified directly under the -`proxy` key. Old service definitions using the nested config will continue to -work and have the values copied into the new location. This allows the upstreams -to be registered centrally rather than being part of the local-only config -passed to the proxy instance. - -For full details of the additional configurable options available when using the -built-in proxy see the [built-in proxy configuration -reference](/consul/docs/connect/configuration). - -### Prepared Query Upstreams - -The upstream destination may also be a -[prepared query](/consul/api-docs/query). -This allows complex service discovery behavior such as connecting to -the nearest neighbor or filtering by tags. - -For example, given a prepared query named "nearest-redis" that is -configured to route to the nearest Redis instance, an upstream can be -configured to route to this query. In the example below, any TCP connection -to port 1234 will attempt a Connect-based connection to the nearest Redis -service. - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "upstreams": [ - { - "destination_name": "redis", - "destination_type": "prepared_query", - "local_bind_port": 1234 - } - ] - } - } - } -} -``` - -For full details of the additional configurable options available when using the -built-in proxy see the [built-in proxy configuration -reference](/consul/docs/connect/configuration). - -### Custom Managed Proxy - -[Custom proxies](/consul/docs/connect/proxies/integrate) can also be -configured to run as a managed proxy. To configure custom proxies, specify -an alternate command to execute for the proxy: - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "exec_mode": "daemon", - "command": ["/usr/bin/my-proxy", "-flag-example"], - "config": { - "foo": "bar" - } - } - } - } -} -``` - -The `exec_mode` value specifies how the proxy is executed. The only -supported value at this time is "daemon". The command is the binary and -any arguments to execute. -The "daemon" mode expects a proxy to run as a long-running, blocking -process. It should not double-fork into the background. The custom -proxy should retrieve its configuration (such as the port to run on) -via the [custom proxy integration APIs](/consul/docs/connect/proxies/integrate). - -The default proxy command can be changed at an agent-global level -in the agent configuration. An example in HCL format is shown below. - -``` -connect { - proxy_defaults { - command = ["/usr/bin/my-proxy"] - } -} -``` - -With this configuration, all services registered without an explicit -proxy command will use `my-proxy` instead of the default built-in proxy. - -The `config` key is an optional opaque JSON object which will be passed through -to the proxy via the proxy configuration endpoint to allow any configuration -options the proxy needs to be specified. See the [built-in proxy -configuration reference](/consul/docs/connect/configuration) -for details of config options that can be passed when using the built-in proxy. - -### Managed Proxy Logs - -Managed proxies have both stdout and stderr captured in log files in the agent's -`data_dir`. They can be found in -`/proxy/logs/-std{err,out}.log`. - -The built-in proxy will inherit its log level from the agent so if the agent is -configured with `log_level = DEBUG`, a proxy it starts will also output `DEBUG` -level logs showing service discovery, certificate and authorization information. - -~> **Note:** In `-dev` mode there is no `data_dir` unless one is explicitly -configured so logging is disabled. You can access logs by providing the -[`-data-dir`](/consul/docs/agent/config/cli-flags#_data_dir) CLI option. If a data dir is -configured, this will also cause proxy processes to stay running when the agent -terminates as described in [Lifecycle](#lifecycle). diff --git a/website/content/docs/connect/registration/index.mdx b/website/content/docs/connect/registration/index.mdx index 24c7a8125192..96db32788776 100644 --- a/website/content/docs/connect/registration/index.mdx +++ b/website/content/docs/connect/registration/index.mdx @@ -7,14 +7,13 @@ description: >- # Service Mesh Proxy Overview -To make Connect aware of proxies you will need to register them in a [service -definition](/consul/docs/discovery/services), just like you would register any other service with Consul. This section outlines your options for registering Connect proxies, either using independent registrations, or in nested sidecar registrations. +To enable service mesh proxies, you must define and register them with Consul. Proxies are a type of service in Consul that facilitate highly secure communication between services in a service mesh. The topics in the section outline your options for registering service mesh proxies. You can register proxies independently or nested inside a sidecar service registration. -## Proxy Service Registration +## Proxy service registration To register proxies with independent proxy service registrations, you can define them in either in config files or via the API just like any other service. Learn more about all of the options you can define when registering your proxy service in the [proxy registration documentation](/consul/docs/connect/registration/service-registration). -## Sidecar Service Registration +## Sidecar service registration To reduce the amount of boilerplate needed for a sidecar proxy, application service definitions may define an inline sidecar service block. This is an opinionated diff --git a/website/content/docs/connect/registration/service-registration.mdx b/website/content/docs/connect/registration/service-registration.mdx index 2bfe409b5b85..666f739fa17c 100644 --- a/website/content/docs/connect/registration/service-registration.mdx +++ b/website/content/docs/connect/registration/service-registration.mdx @@ -9,6 +9,7 @@ description: >- This topic describes how to declare a proxy as a `connect-proxy` in service definitions. The `kind` must be declared and information about the service they represent must be provided to function as a Consul service mesh proxy. + ## Configuration Configure a service mesh proxy using the following syntax: @@ -178,9 +179,7 @@ You can configure the service mesh proxy to create listeners for upstream servic ### Upstream Configuration Examples -Upstreams support multiple destination types. The following examples include information about each implementation. - --> **Snake case**: The examples in this topic use `snake_case` because the syntax is supported in configuration files and API registrations. See [Service Definition Parameter Case](/consul/docs/discovery/services#service-definition-parameter-case) for additional information. +Upstreams support multiple destination types. The following examples include information about each implementation. Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. @@ -300,10 +299,7 @@ The proxy will default to `direct` mode if a mode cannot be determined from the The following examples show additional configuration for transparent proxies. -Added in v1.10.0. - --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). +Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Configure a proxy listener for outbound traffic on port 22500 @@ -328,8 +324,7 @@ registrations](/consul/docs/discovery/services#service-definition-parameter-case The following examples show all possible mesh gateway configurations. --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). + Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Using a Local/Egress Gateway in the Local Datacenter @@ -385,9 +380,8 @@ The following examples show possible configurations to expose HTTP paths through Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). + +Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Expose listeners in Envoy for HTTP and GRPC checks registered with the local Consul agent diff --git a/website/content/docs/connect/registration/sidecar-service.mdx b/website/content/docs/connect/registration/sidecar-service.mdx index adf5ef20f034..cfda3b22ab79 100644 --- a/website/content/docs/connect/registration/sidecar-service.mdx +++ b/website/content/docs/connect/registration/sidecar-service.mdx @@ -7,20 +7,16 @@ description: >- # Register a Service Mesh Proxy in a Service Registration -Connect proxies are typically deployed as "sidecars" that run on the same node -as the single service instance that they handle traffic for. They might be on -the same VM or running as a separate container in the same network namespace. +This topic describes how to declare a proxy as a _sidecar_ proxy. +Sidecar proxies run on the same node as the single service instance that they handle traffic for. +They may be on the same VM or running as a separate container in the same network namespace. -To simplify the configuration experience when deploying a sidecar for a service -instance, Consul 1.3 introduced a new field in the Connect block of the [service -definition](/consul/docs/discovery/services). +## Configuration -The `connect.sidecar_service` field is a complete nested service definition on -which almost any regular service definition field can be set. The exceptions are -[noted below](#limitations). If used, the service definition is treated -identically to another top-level service definition. The value of the nested -definition is that _all fields are optional_ with some opinionated defaults -applied that make setting up a sidecar proxy much simpler. +Add the `connect.sidecar_service` block to your service definition file and specify the parameters to configure sidecar proxy behavior. The `sidecar_service` block is a service definition that can contain most regular service definition fields. Refer to [Limitations](#limitations) for information about unsupported service definition fields for sidecar proxies. + +Consul treats sidecar proxy service definitions as a root-level service definition. All fields are optional in nested +definitions, which default to opinionated settings that are intended to reduce burden of setting up a sidecar proxy. ## Minimal Example @@ -134,7 +130,7 @@ proxy. - `kind` - Defaults to `connect-proxy`. This can't be overridden currently. - `check`, `checks` - By default we add a TCP check on the local address and port for the proxy, and a [service alias - check](/consul/docs/discovery/checks#alias) for the parent service. If either + check](/consul/docs/services/usage/checks#alias-checks) for the parent service. If either `check` or `checks` fields are set, only the provided checks are registered. - `proxy.destination_service_name` - Defaults to the parent service name. - `proxy.destination_service_id` - Defaults to the parent service ID. @@ -143,8 +139,7 @@ proxy. ## Limitations -Almost all fields in a [service definition](/consul/docs/discovery/services) may be -set on the `connect.sidecar_service` except for the following: +The following fields are not supported in the `connect.sidecar_service` block: - `id` - Sidecar services get an ID assigned and it is an error to override this. This ensures the agent can correctly deregister the sidecar service @@ -153,9 +148,6 @@ set on the `connect.sidecar_service` except for the following: unset this to make the registration be for a regular non-connect-proxy service. - `connect.sidecar_service` - Service definitions can't be nested recursively. -- `connect.proxy` - (Deprecated) [Managed - proxies](/consul/docs/connect/proxies/managed-deprecated) can't be defined on a - sidecar. - `connect.native` - Currently the `kind` is fixed to `connect-proxy` and it's an error to register a `connect-proxy` that is also Connect-native. diff --git a/website/content/docs/consul-vs-other/dns-tools-compare.mdx b/website/content/docs/consul-vs-other/dns-tools-compare.mdx index 6aab1440668b..f4d27de8ab45 100644 --- a/website/content/docs/consul-vs-other/dns-tools-compare.mdx +++ b/website/content/docs/consul-vs-other/dns-tools-compare.mdx @@ -12,5 +12,5 @@ description: >- Consul was originally designed as a centralized service registry for any cloud environment that dynamically tracks services as they are added, changed, or removed within a compute infrastructure. Consul maintains a catalog of these registered services and their attributes, such as IP addresses or service name. For more information, refer to [What is Service Discovery?](/consul/docs/concepts/service-discovery). -As a result, Consul can also provide basic DNS functionality, including [lookups, alternate domains, and access controls](/consul/docs/discovery/dns). Since Consul is platform agnostic, you can retrieve service information across both cloud and on-premises data centers. Consul does not natively support some advanced DNS capabilities, such as filters or advanced routing logic. However, you can integrate Consul with existing DNS solutions, such as [NS1](https://help.ns1.com/hc/en-us/articles/360039417093-NS1-Consul-Integration-Overview) and [DNSimple](https://blog.dnsimple.com/2022/05/consul-integration/), to support these advanced capabilities. +As a result, Consul can also provide basic DNS functionality, including [lookups, alternate domains, and access controls](/consul/docs/services/discovery/dns-overview). Since Consul is platform agnostic, you can retrieve service information across both cloud and on-premises data centers. Consul does not natively support some advanced DNS capabilities, such as filters or advanced routing logic. However, you can integrate Consul with existing DNS solutions, such as [NS1](https://help.ns1.com/hc/en-us/articles/360039417093-NS1-Consul-Integration-Overview) and [DNSimple](https://blog.dnsimple.com/2022/05/consul-integration/), to support these advanced capabilities. diff --git a/website/content/docs/discovery/checks.mdx b/website/content/docs/discovery/checks.mdx deleted file mode 100644 index 0c2945602a2c..000000000000 --- a/website/content/docs/discovery/checks.mdx +++ /dev/null @@ -1,885 +0,0 @@ ---- -layout: docs -page_title: Configure Health Checks -description: >- - Agents can be configured to periodically perform custom checks on the health of a service instance or node. Learn about the types of health checks and how to define them in agent and service configuration files. ---- - -# Health Checks - -One of the primary roles of the agent is management of system-level and application-level health -checks. A health check is considered to be application-level if it is associated with a -service. If not associated with a service, the check monitors the health of the entire node. - -Review the [health checks tutorial](/consul/tutorials/developer-discovery/service-registration-health-checks) -to get a more complete example on how to leverage health check capabilities in Consul. - -A check is defined in a configuration file or added at runtime over the HTTP interface. Checks -created via the HTTP interface persist with that node. - -There are several types of checks: - -- [`Script + Interval`](#script-check) - These checks invoke an external application - that performs the health check. - -- [`HTTP + Interval`](#http-check) - These checks make an HTTP `GET` request to the specified URL - in the health check definition. - -- [`TCP + Interval`](#tcp-check) - These checks attempt a TCP connection to the specified - address and port in the health check definition. - -- [`UDP + Interval`](#udp-check) - These checks direct the client to periodically send UDP datagrams - to the specified address and port in the health check definition. - -- [`OSService + Interval`](#osservice-check) - These checks periodically direct the Consul agent to monitor - the health of a service running on the host operating system. - -- [`Time to Live (TTL)`](#time-to-live-ttl-check) - These checks attempt an HTTP connection after a given TTL elapses. - -- [`Docker + Interval`](#docker-check) - These checks invoke an external application that - is packaged within a Docker container. - -- [`gRPC + Interval`](#grpc-check) - These checks are intended for applications that support the standard - [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - -- [`H2ping + Interval`](#h2ping-check) - These checks test an endpoint that uses HTTP/2 - by connecting to the endpoint and sending a ping frame. - -- [`Alias`](#alias-check) - These checks alias the health state of another registered - node or service. - - -## Registering a health check - -There are three ways to register a service with health checks: - -1. Start or reload a Consul agent with a service definition file in the - [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). -1. Call the - [`/agent/service/register`](/consul/api-docs/agent/service#register-service) - HTTP API endpoint to register the service. -1. Use the - [`consul services register`](/consul/commands/services/register) - CLI command to register the service. - -When a service is registered using the HTTP API endpoint or CLI command, -the checks persist in the Consul data folder across Consul agent restarts. - -## Types of checks - -This section describes the available types of health checks you can use to -automatically monitor the health of a service instance or node. - --> **To manually mark a service unhealthy:** Use the maintenance mode - [CLI command](/consul/commands/maint) or - [HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode) - to temporarily remove one or all service instances on a node - from service discovery DNS and HTTP API query results. - -### Script check - -Script checks periodically invoke an external application that performs the health check, -exits with an appropriate exit code, and potentially generates some output. -The specified `interval` determines the time between check invocations. -The output of a script check is limited to 4KB. -Larger outputs are truncated. - -By default, script checks are configured with a timeout equal to 30 seconds. -To configure a custom script check timeout value, -specify the `timeout` field in the check definition. -After reaching the timeout on a Windows system, -Consul waits for any child processes spawned by the script to finish. -After reaching the timeout on other systems, -Consul attempts to force-kill the script and any child processes it spawned. - -Script checks are not enabled by default. -To enable a Consul agent to perform script checks, -use one of the following agent configuration options: - -- [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#_enable_local_script_checks): - Enable script checks defined in local config files. - Script checks registered using the HTTP API are not allowed. -- [`enable_script_checks`](/consul/docs/agent/config/cli-flags#_enable_script_checks): - Enable script checks no matter how they are registered. - - ~> **Security Warning:** - Enabling non-local script checks in some configurations may introduce - a remote execution vulnerability known to be targeted by malware. - We strongly recommend `enable_local_script_checks` instead. - For more information, refer to - [this blog post](https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations). - -The following service definition file snippet is an example -of a script check definition: - - - -```hcl -check = { - id = "mem-util" - name = "Memory utilization" - args = ["/usr/local/bin/check_mem.py", "-limit", "256MB"] - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Memory utilization", - "args": ["/usr/local/bin/check_mem.py", "-limit", "256MB"], - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -#### Check script conventions - -A check script's exit code is used to determine the health check status: - -- Exit code 0 - Check is passing -- Exit code 1 - Check is warning -- Any other code - Check is failing - -Any output of the script is captured and made available in the -`Output` field of checks included in HTTP API responses, -as in this example from the [local service health endpoint](/consul/api-docs/agent/service#by-name-json). - -### HTTP check - -HTTP checks periodically make an HTTP `GET` request to the specified URL, -waiting the specified `interval` amount of time between requests. -The status of the service depends on the HTTP response code: any `2xx` code is -considered passing, a `429 Too ManyRequests` is a warning, and anything else is -a failure. This type of check -should be preferred over a script that uses `curl` or another external process -to check a simple HTTP operation. By default, HTTP checks are `GET` requests -unless the `method` field specifies a different method. Additional request -headers can be set through the `header` field which is a map of lists of -strings, such as `{"x-foo": ["bar", "baz"]}`. - -By default, HTTP checks are configured with a request timeout equal to 10 seconds. -To configure a custom HTTP check timeout value, -specify the `timeout` field in the check definition. -The output of an HTTP check is limited to approximately 4KB. -Larger outputs are truncated. -HTTP checks also support TLS. By default, a valid TLS certificate is expected. -Certificate verification can be turned off by setting the `tls_skip_verify` -field to `true` in the check definition. When using TLS, the SNI is implicitly -determined from the URL if it uses a hostname instead of an IP address. -You can explicitly set the SNI value by setting `tls_server_name`. - -Consul follows HTTP redirects by default. -To disable redirects, set the `disable_redirects` field to `true`. - -The following service definition file snippet is an example -of an HTTP check definition: - - - -```hcl -check = { - id = "api" - name = "HTTP API on port 5000" - http = "https://localhost:5000/health" - tls_server_name = "" - tls_skip_verify = false - method = "POST" - header = { - Content-Type = ["application/json"] - } - body = "{\"method\":\"health\"}" - disable_redirects = true - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "api", - "name": "HTTP API on port 5000", - "http": "https://localhost:5000/health", - "tls_server_name": "", - "tls_skip_verify": false, - "method": "POST", - "header": { "Content-Type": ["application/json"] }, - "body": "{\"method\":\"health\"}", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### TCP check - -TCP checks periodically make a TCP connection attempt to the specified IP/hostname and port, waiting `interval` amount of time between attempts. -If no hostname is specified, it defaults to "localhost". -The health check status is `success` if the target host accepts the connection attempt, -otherwise the status is `critical`. In the case of a hostname that -resolves to both IPv4 and IPv6 addresses, an attempt is made to both -addresses, and the first successful connection attempt results in a -successful check. This type of check should be preferred over a script that -uses `netcat` or another external process to check a simple socket operation. - -By default, TCP checks are configured with a request timeout equal to 10 seconds. -To configure a custom TCP check timeout value, -specify the `timeout` field in the check definition. - -The following service definition file snippet is an example -of a TCP check definition: - - - -```hcl -check = { - id = "ssh" - name = "SSH TCP on port 22" - tcp = "localhost:22" - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "ssh", - "name": "SSH TCP on port 22", - "tcp": "localhost:22", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### UDP check - -UDP checks periodically direct the Consul agent to send UDP datagrams -to the specified IP/hostname and port, -waiting `interval` amount of time between attempts. -The check status is set to `success` if any response is received from the targeted UDP server. -Any other result sets the status to `critical`. - -By default, UDP checks are configured with a request timeout equal to 10 seconds. -To configure a custom UDP check timeout value, -specify the `timeout` field in the check definition. -If any timeout on read exists, the check is still considered healthy. - -The following service definition file snippet is an example -of a UDP check definition: - - - -```hcl -check = { - id = "dns" - name = "DNS UDP on port 53" - udp = "localhost:53" - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "dns", - "name": "DNS UDP on port 53", - "udp": "localhost:53", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### OSService check - -OSService checks periodically direct the Consul agent to monitor the health of a service running on -the host operating system as either a Windows service (Windows) or a SystemD service (Unix). -The check is logged as `healthy` if the service is running. -If it is stopped or not running, the status is `critical`. All other results set -the status to `warning`, which indicates that the check is not reliable because -an issue is preventing the check from determining the health of the service. - -The following service definition file snippet is an example -of an OSService check definition: - - - -```hcl -check = { - id = "myco-svctype-svcname-001" - name = "svcname-001 Windows Service Health" - service_id = "flash_pnl_1" - os_service = "myco-svctype-svcname-001" - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "myco-svctype-svcname-001", - "name": "svcname-001 Windows Service Health", - "service_id": "flash_pnl_1", - "os_service": "myco-svctype-svcname-001", - "interval": "10s" - } -} -``` - - - -### Time to live (TTL) check - -TTL checks retain their last known state for the specified `ttl` duration. -The state of the check updates periodically over the HTTP interface. -If the `ttl` duration elapses before a new check update -is provided over the HTTP interface, -the check is set to `critical` state. - -This mechanism relies on the application to directly report its health. -For example, a healthy app can periodically `PUT` a status update to the HTTP endpoint. -Then, if the app is disrupted and unable to perform this update -before the TTL expires, the health check enters the `critical` state. -The endpoints used to update health information for a given check are: [pass](/consul/api-docs/agent/check#ttl-check-pass), -[warn](/consul/api-docs/agent/check#ttl-check-warn), [fail](/consul/api-docs/agent/check#ttl-check-fail), -and [update](/consul/api-docs/agent/check#ttl-check-update). TTL checks also persist their -last known status to disk. This persistence allows the Consul agent to restore the last known -status of the check across agent restarts. Persisted check status is valid through the -end of the TTL from the time of the last check. - -To manually mark a service unhealthy, -it is far more convenient to use the maintenance mode -[CLI command](/consul/commands/maint) or -[HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode) -rather than a TTL health check with arbitrarily high `ttl`. - -The following service definition file snippet is an example -of a TTL check definition: - - - -```hcl -check = { - id = "web-app" - name = "Web App Status" - notes = "Web app does a curl internally every 10 seconds" - ttl = "30s" -} -``` - -```json -{ - "check": { - "id": "web-app", - "name": "Web App Status", - "notes": "Web app does a curl internally every 10 seconds", - "ttl": "30s" - } -} -``` - - - -### Docker check - -These checks depend on periodically invoking an external application that -is packaged within a Docker Container. The application is triggered within the running -container through the Docker Exec API. We expect that the Consul agent user has access -to either the Docker HTTP API or the unix socket. Consul uses `$DOCKER_HOST` to -determine the Docker API endpoint. The application is expected to run, perform a health -check of the service running inside the container, and exit with an appropriate exit code. -The check should be paired with an invocation interval. The shell on which the check -has to be performed is configurable, making it possible to run containers which -have different shells on the same host. -The output of a Docker check is limited to 4KB. -Larger outputs are truncated. - -Docker checks are not enabled by default. -To enable a Consul agent to perform Docker checks, -use one of the following agent configuration options: - -- [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#_enable_local_script_checks): - Enable script checks defined in local config files. - Script checks registered using the HTTP API are not allowed. - -- [`enable_script_checks`](/consul/docs/agent/config/cli-flags#_enable_script_checks): - Enable script checks no matter how they are registered. - - !> **Security Warning:** - We recommend using `enable_local_script_checks` instead of `enable_script_checks` in production - environments, as remote script checks are more vulnerable to malware attacks. Learn more about how [script checks can be exploited](https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations#how-script-checks-can-be-exploited). - -The following service definition file snippet is an example -of a Docker check definition: - - - -```hcl -check = { - id = "mem-util" - name = "Memory utilization" - docker_container_id = "f972c95ebf0e" - shell = "/bin/bash" - args = ["/usr/local/bin/check_mem.py"] - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Memory utilization", - "docker_container_id": "f972c95ebf0e", - "shell": "/bin/bash", - "args": ["/usr/local/bin/check_mem.py"], - "interval": "10s" - } -} -``` - - - -### gRPC check - -gRPC checks are intended for applications that support the standard -[gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). -The state of the check will be updated by periodically probing the configured endpoint, -waiting `interval` amount of time between attempts. - -By default, gRPC checks are configured with a timeout equal to 10 seconds. -To configure a custom Docker check timeout value, -specify the `timeout` field in the check definition. - -gRPC checks default to not using TLS. -To enable TLS, set `grpc_use_tls` in the check definition. -If TLS is enabled, then by default, a valid TLS certificate is expected. -Certificate verification can be turned off by setting the -`tls_skip_verify` field to `true` in the check definition. -To check on a specific service instead of the whole gRPC server, -add the service identifier after the `gRPC` check's endpoint. - -The following example shows a gRPC check for a whole application: - - - -```hcl -check = { - id = "mem-util" - name = "Service health status" - grpc = "127.0.0.1:12345" - grpc_use_tls = true - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Service health status", - "grpc": "127.0.0.1:12345", - "grpc_use_tls": true, - "interval": "10s" - } -} -``` - - - -The following example shows a gRPC check for the specific `my_service` service: - - - -```hcl -check = { - id = "mem-util" - name = "Service health status" - grpc = "127.0.0.1:12345/my_service" - grpc_use_tls = true - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Service health status", - "grpc": "127.0.0.1:12345/my_service", - "grpc_use_tls": true, - "interval": "10s" - } -} -``` - - - -### H2ping check - -H2ping checks test an endpoint that uses http2 by connecting to the endpoint -and sending a ping frame, waiting `interval` amount of time between attempts. -If the ping is successful within a specified timeout, -then the check status is set to `success`. - -By default, h2ping checks are configured with a request timeout equal to 10 seconds. -To configure a custom h2ping check timeout value, -specify the `timeout` field in the check definition. - -TLS is enabled by default. -To disable TLS and use h2c, set `h2ping_use_tls` to `false`. -If TLS is not disabled, a valid certificate is required unless `tls_skip_verify` is set to `true`. - -The following service definition file snippet is an example -of an h2ping check definition: - - - -```hcl -check = { - id = "h2ping-check" - name = "h2ping" - h2ping = "localhost:22222" - interval = "10s" - h2ping_use_tls = false -} -``` - -```json -{ - "check": { - "id": "h2ping-check", - "name": "h2ping", - "h2ping": "localhost:22222", - "interval": "10s", - "h2ping_use_tls": false - } -} -``` - - - -### Alias check - -These checks alias the health state of another registered -node or service. The state of the check updates asynchronously, but is -nearly instant. For aliased services on the same agent, the local state is monitored -and no additional network resources are consumed. For other services and nodes, -the check maintains a blocking query over the agent's connection with a current -server and allows stale requests. If there are any errors in watching the aliased -node or service, the check state is set to `critical`. -For the blocking query, the check uses the ACL token set on the service or check definition. -If no ACL token is set in the service or check definition, -the blocking query uses the agent's default ACL token -([`acl.tokens.default`](/consul/docs/agent/config/config-files#acl_tokens_default)). - -~> **Configuration info**: The alias check configuration expects the alias to be -registered on the same agent as the one you are aliasing. If the service is -not registered with the same agent, `"alias_node": ""` must also be -specified. When using `alias_node`, if no service is specified, the check will -alias the health of the node. If a service is specified, the check will alias -the specified service on this particular node. - -The following service definition file snippet is an example -of an alias check for a local service: - - - -```hcl -check = { - id = "web-alias" - alias_service = "web" -} -``` - -```json -{ - "check": { - "id": "web-alias", - "alias_service": "web" - } -} -``` - - - -## Check definition - -This section covers some of the most common options for check definitions. -For a complete list of all check options, refer to the -[Register Check HTTP API endpoint documentation](/consul/api-docs/agent/check#json-request-body-schema). - --> **Casing for check options:** - The correct casing for an option depends on whether the check is defined in - a service definition file or an HTTP API JSON request body. - For example, the option `deregister_critical_service_after` in a service - definition file is instead named `DeregisterCriticalServiceAfter` in an - HTTP API JSON request body. - -#### General options - -- `name` `(string: )` - Specifies the name of the check. - -- `id` `(string: "")` - Specifies a unique ID for this check on this node. - - If unspecified, Consul defines the check id by: - - If the check definition is embedded within a service definition file, - a unique check id is auto-generated. - - Otherwise, the `id` is set to the value of `name`. - If names might conflict, you must provide unique IDs to avoid - overwriting existing checks with the same id on this node. - -- `interval` `(string: )` - Specifies - the frequency at which to run this check. - Required for all check types except TTL and alias checks. - - The value is parsed by Go's `time` package, and has the following - [formatting specification](https://golang.org/pkg/time/#ParseDuration): - - > A duration string is a possibly signed sequence of decimal numbers, each with - > optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". - > Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - -- `service_id` `(string: )` - Specifies - the ID of a service instance to associate this check with. - That service instance must be on this node. - If not specified, this check is treated as a node-level check. - For more information, refer to the - [service-bound checks](#service-bound-checks) section. - -- `status` `(string: "")` - Specifies the initial status of the health check as - "critical" (default), "warning", or "passing". For more details, refer to - the [initial health check status](#initial-health-check-status) section. - - -> **Health defaults to critical:** If health status it not initially specified, - it defaults to "critical" to protect against including a service - in discovery results before it is ready. - -- `deregister_critical_service_after` `(string: "")` - If specified, - the associated service and all its checks are deregistered - after this check is in the critical state for more than the specified value. - The value has the same formatting specification as the [`interval`](#interval) field. - - The minimum timeout is 1 minute, - and the process that reaps critical services runs every 30 seconds, - so it may take slightly longer than the configured timeout to trigger the deregistration. - This field should generally be configured with a timeout that's significantly longer than - any expected recoverable outage for the given service. - -- `notes` `(string: "")` - Provides a human-readable description of the check. - This field is opaque to Consul and can be used however is useful to the user. - For example, it could be used to describe the current state of the check. - -- `token` `(string: "")` - Specifies an ACL token used for any interaction - with the catalog for the check, including - [anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - - For alias checks, this token is used if a remote blocking query is necessary to watch the state of the aliased node or service. - -#### Success/failures before passing/warning/critical - -To prevent flapping health checks and limit the load they cause on the cluster, -a health check may be configured to become passing/warning/critical only after a -specified number of consecutive checks return as passing/critical. -The status does not transition states until the configured threshold is reached. - -- `success_before_passing` - Number of consecutive successful results required - before check status transitions to passing. Defaults to `0`. Added in Consul 1.7.0. - -- `failures_before_warning` - Number of consecutive unsuccessful results required - before check status transitions to warning. Defaults to the same value as that of - `failures_before_critical` to maintain the expected behavior of not changing the - status of service checks to `warning` before `critical` unless configured to do so. - Values higher than `failures_before_critical` are invalid. Added in Consul 1.11.0. - -- `failures_before_critical` - Number of consecutive unsuccessful results required - before check status transitions to critical. Defaults to `0`. Added in Consul 1.7.0. - -This feature is available for all check types except TTL and alias checks. -By default, both passing and critical thresholds are set to 0 so the check -status always reflects the last check result. - - - -```hcl -checks = [ - { - name = "HTTP TCP on port 80" - tcp = "localhost:80" - interval = "10s" - timeout = "1s" - success_before_passing = 3 - failures_before_warning = 1 - failures_before_critical = 3 - } -] -``` - -```json -{ - "checks": [ - { - "name": "HTTP TCP on port 80", - "tcp": "localhost:80", - "interval": "10s", - "timeout": "1s", - "success_before_passing": 3, - "failures_before_warning": 1, - "failures_before_critical": 3 - } - ] -} -``` - - - -## Initial health check status - -By default, when checks are registered against a Consul agent, the state is set -immediately to "critical". This is useful to prevent services from being -registered as "passing" and entering the service pool before they are confirmed -to be healthy. In certain cases, it may be desirable to specify the initial -state of a health check. This can be done by specifying the `status` field in a -health check definition, like so: - - - -```hcl -check = { - id = "mem" - args = ["/bin/check_mem", "-limit", "256MB"] - interval = "10s" - status = "passing" -} -``` - -```json -{ - "check": { - "id": "mem", - "args": ["/bin/check_mem", "-limit", "256MB"], - "interval": "10s", - "status": "passing" - } -} -``` - - - -The above service definition would cause the new "mem" check to be -registered with its initial state set to "passing". - -## Service-bound checks - -Health checks may optionally be bound to a specific service. This ensures -that the status of the health check will only affect the health status of the -given service instead of the entire node. Service-bound health checks may be -provided by adding a `service_id` field to a check configuration: - - - -```hcl -check = { - id = "web-app" - name = "Web App Status" - service_id = "web-app" - ttl = "30s" -} -``` - -```json -{ - "check": { - "id": "web-app", - "name": "Web App Status", - "service_id": "web-app", - "ttl": "30s" - } -} -``` - - - -In the above configuration, if the web-app health check begins failing, it will -only affect the availability of the web-app service. All other services -provided by the node will remain unchanged. - -## Agent certificates for TLS checks - -The [enable_agent_tls_for_checks](/consul/docs/agent/config/config-files#enable_agent_tls_for_checks) -agent configuration option can be utilized to have HTTP or gRPC health checks -to use the agent's credentials when configured for TLS. - -## Multiple check definitions - -Multiple check definitions can be defined using the `checks` (plural) -key in your configuration file. - - - -```hcl -checks = [ - { - id = "chk1" - name = "mem" - args = ["/bin/check_mem", "-limit", "256MB"] - interval = "5s" - }, - { - id = "chk2" - name = "/health" - http = "http://localhost:5000/health" - interval = "15s" - }, - { - id = "chk3" - name = "cpu" - args = ["/bin/check_cpu"] - interval = "10s" - }, - ... -] -``` - -```json -{ - "checks": [ - { - "id": "chk1", - "name": "mem", - "args": ["/bin/check_mem", "-limit", "256MB"], - "interval": "5s" - }, - { - "id": "chk2", - "name": "/health", - "http": "http://localhost:5000/health", - "interval": "15s" - }, - { - "id": "chk3", - "name": "cpu", - "args": ["/bin/check_cpu"], - "interval": "10s" - }, - ... - ] -} -``` - - diff --git a/website/content/docs/discovery/dns.mdx b/website/content/docs/discovery/dns.mdx deleted file mode 100644 index 90038ae2955a..000000000000 --- a/website/content/docs/discovery/dns.mdx +++ /dev/null @@ -1,594 +0,0 @@ ---- -layout: docs -page_title: Find services with DNS -description: >- - For service discovery use cases, Domain Name Service (DNS) is the main interface to look up, query, and address Consul nodes and services. Learn how a Consul DNS lookup can help you find services by tag, name, namespace, partition, datacenter, or domain. ---- - -# Query services with DNS - -One of the primary query interfaces for Consul is DNS. -The DNS interface allows applications to make use of service -discovery without any high-touch integration with Consul. - -For example, instead of making HTTP API requests to Consul, -a host can use the DNS server directly via name lookups -like `redis.service.us-east-1.consul`. This query automatically -translates to a lookup of nodes that provide the `redis` service, -are located in the `us-east-1` datacenter, and have no failing health checks. -It's that simple! - -There are a number of configuration options that are important for the DNS interface, -specifically [`client_addr`](/consul/docs/agent/config/config-files#client_addr),[`ports.dns`](/consul/docs/agent/config/config-files#dns_port), -[`recursors`](/consul/docs/agent/config/config-files#recursors),[`domain`](/consul/docs/agent/config/config-files#domain), -[`alt_domain`](/consul/docs/agent/config/config-files#alt_domain), and [`dns_config`](/consul/docs/agent/config/config-files#dns_config). -By default, Consul will listen on 127.0.0.1:8600 for DNS queries in the `consul.` -domain, without support for further DNS recursion. Please consult the -[documentation on configuration options](/consul/docs/agent/config), -specifically the configuration items linked above, for more details. - -There are a few ways to use the DNS interface. One option is to use a custom -DNS resolver library and point it at Consul. Another option is to set Consul -as the DNS server for a node and provide a -[`recursors`](/consul/docs/agent/config/config-files#recursors) configuration so that non-Consul queries -can also be resolved. The last method is to forward all queries for the "consul." -domain to a Consul agent from the existing DNS server. Review the -[DNS Forwarding tutorial](/consul/tutorials/networking/dns-forwarding?utm_source=docs) for examples. - -You can experiment with Consul's DNS server on the command line using tools such as `dig`: - -```shell-session -$ dig @127.0.0.1 -p 8600 redis.service.dc1.consul. ANY -``` - --> **Note:** In DNS, all queries are case-insensitive. A lookup of `PostgreSQL.node.dc1.consul` will find all nodes named `postgresql`. - -## Node Lookups - -To resolve names, Consul relies on a very specific format for queries. -There are fundamentally two types of queries: node lookups and service lookups. -A node lookup, a simple query for the address of a named node, looks like this: - -```text -.node[.]. -``` - -For example, if we have a `foo` node with default settings, we could -look for `foo.node.dc1.consul.` The datacenter is an optional part of -the FQDN: if not provided, it defaults to the datacenter of the agent. -If we know `foo` is running in the same datacenter as our local agent, -we can instead use `foo.node.consul.` This convention allows for terse -syntax where appropriate while supporting queries of nodes in remote -datacenters as necessary. - -For a node lookup, the only records returned are A and AAAA records -containing the IP address, and TXT records containing the -`node_meta` values of the node. - -```shell-session -$ dig @127.0.0.1 -p 8600 foo.node.consul ANY - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 foo.node.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 24355 -;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 0 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;foo.node.consul. IN ANY - -;; ANSWER SECTION: -foo.node.consul. 0 IN A 10.1.10.12 -foo.node.consul. 0 IN TXT "meta_key=meta_value" -foo.node.consul. 0 IN TXT "value only" - - -;; AUTHORITY SECTION: -consul. 0 IN SOA ns.consul. postmaster.consul. 1392836399 3600 600 86400 0 -``` - -By default the TXT records value will match the node's metadata key-value -pairs according to [RFC1464](https://www.ietf.org/rfc/rfc1464.txt). -Alternatively, the TXT record will only include the node's metadata value when the -node's metadata key starts with `rfc1035-`. - - -### Node Lookups for Consul Enterprise - -Consul nodes exist at the admin partition level within a datacenter. -By default, the partition and datacenter used in a [node lookup](#node-lookups) are -the partition and datacenter of the Consul agent that received the DNS query. - -Use the following query format to specify a partition for a node lookup: -```text -.node..ap..dc. -``` - -Consul server agents are in the `default` partition. -If DNS queries are addressed to Consul server agents, -node lookups to non-`default` partitions must explicitly specify -the partition of the target node. - -## Service Lookups - -A service lookup is used to query for service providers. Service queries support -two lookup methods: standard and strict [RFC 2782](https://tools.ietf.org/html/rfc2782). - -By default, SRV weights are all set at 1, but changing weights is supported using the -`Weights` attribute of the [service definition](/consul/docs/discovery/services). - -Note that DNS is limited in size per request, even when performing DNS TCP -queries. - -For services having many instances (more than 500), it might not be possible to -retrieve the complete list of instances for the service. - -When DNS SRV response are sent, order is randomized, but weights are not -taken into account. In the case of truncation different clients using weighted SRV -responses will have partial and inconsistent views of instances weights so the -request distribution could be skewed from the intended weights. In that case, -it is recommended to use the HTTP API to retrieve the list of nodes. - -### Standard Lookup - -The format of a standard service lookup is: - -```text -[.].service[.]. -``` - -The `tag` is optional, and, as with node lookups, the `datacenter` is as -well. If no tag is provided, no filtering is done on tag. If no -datacenter is provided, the datacenter of this Consul agent is assumed. - -If we want to find any redis service providers in our local datacenter, -we could query `redis.service.consul.` If we want to find the PostgreSQL -primary in a particular datacenter, we could query -`primary.postgresql.service.dc2.consul.` - -The DNS query system makes use of health check information to prevent routing -to unhealthy nodes. When a service query is made, any services failing their health -check or failing a node system check will be omitted from the results. To allow -for simple load balancing, the set of nodes returned is also randomized each time. -These mechanisms make it easy to use DNS along with application-level retries -as the foundation for an auto-healing service oriented architecture. - -For standard services queries, both A and SRV records are supported. SRV records -provide the port that a service is registered on, enabling clients to avoid relying -on well-known ports. SRV records are only served if the client specifically requests -them, like so: - -```shell-session -$ dig @127.0.0.1 -p 8600 consul.service.consul SRV - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 consul.service.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 -;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;consul.service.consul. IN SRV - -;; ANSWER SECTION: -consul.service.consul. 0 IN SRV 1 1 8300 foobar.node.dc1.consul. - -;; ADDITIONAL SECTION: -foobar.node.dc1.consul. 0 IN A 10.1.10.12 -``` - -### RFC 2782 Lookup - -Valid formats for RFC 2782 SRV lookups depend on -whether you want to filter results based on a service tag: - -- No filtering on service tag: - - ```text - _._tcp[.service][.]. - ``` - -- Filtering on service tag specified in the RFC 2782 protocol field: - - ```text - _._[.service][.]. - ``` - -Per [RFC 2782](https://tools.ietf.org/html/rfc2782), SRV queries must -prepend an underscore (`_`) to the `service` and `protocol` values in a query to -prevent DNS collisions. -To perform no tag-based filtering, specify `tcp` in the RFC 2782 protocol field. -To filter results on a service tag, specify the tag in the RFC 2782 protocol field. - -Other than the query format and default `tcp` protocol/tag value, the behavior -of the RFC style lookup is the same as the standard style of lookup. - -If you registered the service `rabbitmq` on port 5672 and tagged it with `amqp`, -you could make an RFC 2782 query for its SRV record as `_rabbitmq._amqp.service.consul`: - -```shell-session -$ dig @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul SRV - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 52838 -;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;_rabbitmq._amqp.service.consul. IN SRV - -;; ANSWER SECTION: -_rabbitmq._amqp.service.consul. 0 IN SRV 1 1 5672 rabbitmq.node1.dc1.consul. - -;; ADDITIONAL SECTION: -rabbitmq.node1.dc1.consul. 0 IN A 10.1.11.20 -``` - -Again, note that the SRV record returns the port of the service as well as its IP. - -#### SRV response for hosts in the .addr subdomain - -If a service registered to Consul has an explicit IP [`address`](/consul/api-docs/agent/service#address) -or tagged address(es) defined on the service registration, the hostname returned -in the target field of the answer section for the DNS SRV query for the service -will be in the format of `.addr..consul`. - - - - - -In the example below, the `rabbitmq` service has been registered with an explicit -IPv4 address of `192.0.2.10`. - - - -```hcl -node_name = "node1" - -services { - name = "rabbitmq" - address = "192.0.2.10" - port = 5672 -} -``` - -```json -{ - "node_name": "node1", - "services": [ - { - "name": "rabbitmq", - "address": "192.0.2.10", - "port": 5672 - } - ] -} -``` - - - -When performing an SRV query for this service, the SRV response contains a single -record with a hostname in the format of `.addr..consul`. - -```shell-session -$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short -1 1 5672 c000020a.addr.dc1.consul. -``` - -In this example, the hex-encoded IP from the returned hostname is `c000020a`. -Converting each hex octet to decimal reveals the IP address that was specified -in the service registration. - -```shell-session -$ echo -n "c000020a" | perl -ne 'printf("%vd\n", pack("H*", $_))' -192.0.2.10 -``` - - - - - -In the example below, the `rabbitmq` service has been registered with an explicit -IPv6 address of `2001:db8:1:2:cafe::1337`. - - - -```hcl -node_name = "node1" - -services { - name = "rabbitmq" - address = "2001:db8:1:2:cafe::1337" - port = 5672 -} -``` - -```json -{ - "node_name": "node1", - "services": [ - { - "name": "rabbitmq", - "address": "2001:db8:1:2:cafe::1337", - "port": 5672 - } - ] -} -``` - - - -When performing an SRV query for this service, the SRV response contains a single -record with a hostname in the format of `.addr..consul`. - -```shell-session -$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short -1 1 5672 20010db800010002cafe000000001337.addr.dc1.consul. -``` - -In this example, the hex-encoded IP from the returned hostname is -`20010db800010002cafe000000001337`. This is the fully expanded IPv6 address with -colon separators removed. - -The following command re-adds the colon separators to display the fully expanded -IPv6 address that was specified in the service registration. - -```shell-session -$ echo -n "20010db800010002cafe000000001337" | perl -ne 'printf join(":", unpack("(A4)*", $_))."\n"' -2001:0db8:0001:0002:cafe:0000:0000:1337 -``` - - - - - -### Service Lookups for Consul Enterprise - -By default, all service lookups use the `default` namespace -within the partition and datacenter of the Consul agent that received the DNS query. -To lookup services in another namespace, partition, and/or datacenter, -use the [canonical format](#canonical-format). - -Consul server agents are in the `default` partition. -If DNS queries are addressed to Consul server agents, -service lookups to non-`default` partitions must explicitly specify -the partition of the target service. - -To lookup services imported from a cluster peer, -refer to [service virtual IP lookups for Consul Enterprise](#service-virtual-ip-lookups-for-consul-enterprise) instead. - -#### Canonical format - -Use the following query format to specify namespace, partition, and/or datacenter -for `.service`, `.connect`, `.virtual`, and `.ingress` service lookup types. -All three fields (`namespace`, `partition`, `datacenter`) are optional. -```text -[.].service[..ns][..ap][..dc] -``` - -#### Alternative formats for specifying namespace - -Though the [canonical format](#canonical-format) is recommended for readability, -you can use the following query formats specify namespace but not partition: - -- Specify both namespace and datacenter: - - ```text - [.].service... - ``` - -- **Deprecated in Consul 1.11:** - Specify namespace without a datacenter, - which requires that DNS queries are addressed to a Consul agent with - [`dns_config.prefer_namespace`](/consul/docs/agent/config/config-files#dns_prefer_namespace) - set to `true`: - - ```text - [.].service.. - ``` - -### Prepared Query Lookups - -The following formats are valid for prepared query lookups: - -- Standard lookup - - ```text - .query[.]. - ``` - -- [RFC 2782](https://tools.ietf.org/html/rfc2782) SRV lookup - - ```text - _._tcp.query[.]. - ``` - -The `datacenter` is optional, and if not provided, the datacenter of this Consul -agent is assumed. - -The `query name or id` is the given name or ID of an existing -[Prepared Query](/consul/api-docs/query). These behave like standard service -queries but provide a much richer set of features, such as filtering by multiple -tags and automatically failing over to look for services in remote datacenters if -no healthy nodes are available in the local datacenter. Consul 0.6.4 and later also -added support for [prepared query templates](/consul/api-docs/query#prepared-query-templates) -which can match names using a prefix match, allowing one template to apply to -potentially many services. - -To allow for simple load balancing, the set of nodes returned is randomized each time. -Both A and SRV records are supported. SRV records provide the port that a service is -registered on, enabling clients to avoid relying on well-known ports. SRV records are -only served if the client specifically requests them. - -### Connect-Capable Service Lookups - -To find Connect-capable services: - -```text -.connect. -``` - -This will find all [Connect-capable](/consul/docs/connect) -endpoints for the given `service`. A Connect-capable endpoint may be -both a proxy for a service or a natively integrated Connect application. -The DNS interface does not differentiate the two. - -Most services will use a [proxy](/consul/docs/connect/proxies) that handles -service discovery automatically and therefore won't use this DNS format. -This DNS format is primarily useful for [Connect-native](/consul/docs/connect/native) -applications. - -This endpoint currently only finds services within the same datacenter -and doesn't support tags. This DNS interface will be expanded over time. -If you need more complex behavior, please use the -[catalog API](/consul/api-docs/catalog). - -### Service Virtual IP Lookups - -To find the unique virtual IP allocated for a service: - -```text -.virtual[.]. -``` - -This will return the unique virtual IP for any [Connect-capable](/consul/docs/connect) -service. Each Connect service has a virtual IP assigned to it by Consul - this is used -by sidecar proxies for the [Transparent Proxy](/consul/docs/connect/transparent-proxy) feature. -The peer name is an optional part of the FQDN, and it is used to query for the virtual IP -of a service imported from that peer. - -The virtual IP is also added to the service's [Tagged Addresses](/consul/docs/discovery/services#tagged-addresses) -under the `consul-virtual` tag. - -#### Service Virtual IP Lookups for Consul Enterprise - -By default, a service virtual IP lookup uses the `default` namespace -within the partition and datacenter of the Consul agent that received the DNS query. - -To lookup services imported from a cluster peered partition or open-source datacenter, -specify the namespace and peer name in the lookup: -```text -.virtual[.].. -``` - -To lookup services not imported from a cluster peer, -refer to [service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise) instead. - -### Ingress Service Lookups - -To find ingress-enabled services: - -```text -.ingress. -``` - -This will find all [ingress gateway](/consul/docs/connect/gateways/ingress-gateway) -endpoints for the given `service`. - -This endpoint currently only finds services within the same datacenter -and doesn't support tags. This DNS interface will be expanded over time. -If you need more complex behavior, please use the -[catalog API](/consul/api-docs/catalog). - -### UDP Based DNS Queries - -When the DNS query is performed using UDP, Consul will truncate the results -without setting the truncate bit. This is to prevent a redundant lookup over -TCP that generates additional load. If the lookup is done over TCP, the results -are not truncated. - -## Alternative Domain - -By default, Consul responds to DNS queries in the `consul` domain, -but you can set a specific domain for responding to DNS queries by configuring the [`domain`](/consul/docs/agent/config/config-files#domain) parameter. - -In some instances, Consul may need to respond to queries in more than one domain, -such as during a DNS migration or to distinguish between internal and external queries. - -Consul versions 1.5.2+ can be configured to respond to DNS queries on an alternative domain -through the [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) agent configuration -option. As of Consul versions 1.11.0+, Consul's DNS response will use the same domain as was used in the query; -in prior versions, the response may use the primary [`domain`](/consul/docs/agent/config/config-files#domain) no matter which -domain was used in the query. - -In the following example, the `alt_domain` parameter is set to `test-domain`: - -```hcl - alt_domain = "test-domain" -``` - -```shell-session -$ dig @127.0.0.1 -p 8600 consul.service.test-domain SRV -``` - -The following responses are returned: - -``` -;; QUESTION SECTION: -;consul.service.test-domain. IN SRV - -;; ANSWER SECTION: -consul.service.test-domain. 0 IN SRV 1 1 8300 machine.node.dc1.test-domain. - -;; ADDITIONAL SECTION: -machine.node.dc1.test-domain. 0 IN A 127.0.0.1 -machine.node.dc1.test-domain. 0 IN TXT "consul-network-segment=" -``` - --> **PTR queries:** Responses to PTR queries (`.in-addr.arpa.`) will always use the -[primary domain](/consul/docs/agent/config/config-files#domain) (not the alternative domain), -as there is no way for the query to specify a domain. - -## Caching - -By default, all DNS results served by Consul set a 0 TTL value. This disables -caching of DNS results. However, there are many situations in which caching is -desirable for performance and scalability. This is discussed more in the tutorial -for [DNS caching](/consul/tutorials/networking/dns-caching). - -## WAN Address Translation - -By default, Consul DNS queries will return a node's local address, even when -being queried from a remote datacenter. If you need to use a different address -to reach a node from outside its datacenter, you can configure this behavior -using the [`advertise-wan`](/consul/docs/agent/config/cli-flags#_advertise-wan) and -[`translate_wan_addrs`](/consul/docs/agent/config/config-files#translate_wan_addrs) configuration -options. - -## DNS with ACLs - -In order to use the DNS interface when -[Access Control Lists (ACLs)](/consul/docs/security/acl) -are enabled, you must first create ACL tokens with the necessary policies. - -Consul agents resolve DNS requests using one of the preconfigured tokens below, -listed in order of precedence: - -1. The agent's [`default` token](/consul/docs/agent/config/config-files#acl_tokens_default). -2. The built-in [`anonymous` token](/consul/docs/security/acl/acl-tokens#built-in-tokens). - Because the anonymous token is used when any request is made to Consul without - explicitly specifying a token, production deployments should not apply policies - needed for DNS to this token. - -Consul will either accept or deny the request depending on whether the token -has the appropriate authorization. The following table describes the available -DNS lookups and required policies when ACLs are enabled: - -| Lookup | Type | Description | ACLs Required | -| ------------------------------------------------------------------------------ | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `*.node.consul` | [Node](#node-lookups) | Allow resolving DNS requests for the target node (i.e., `.node.consul`) | [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | -| `*.service.consul`, `*.connect.consul`, `*.ingress.consul`, `*.virtual.consul` | [Service: standard](#service-lookups) | Allow resolving DNS requests for target service (e.g., `.service.consul`) instances running on ACL-authorized nodes | [`service:read`](/consul/docs/security/acl/acl-rules#service-rules), [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | -| `*.query.consul` | [Service: prepared query](#prepared-query-lookups) | Allow resolving DNS requests for [service instances specified](/consul/api-docs/query#service-1) by the target prepared query (i.e., `.query.consul`) running on ACL-authorized nodes | [`query:read`](/consul/docs/security/acl/acl-rules#prepared-query-rules), [`service:read`](/consul/docs/security/acl/acl-rules#service-rules), [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | - -For guidance on how to configure an appropriate token for DNS, refer to the -securing Consul with ACLs guides for: - -- [Production Environments](/consul/tutorials/security/access-control-setup-production#token-for-dns) -- [Development Environments](/consul/tutorials/day-0/access-control-setup?utm_source=docs#enable-acls-on-consul-clients) diff --git a/website/content/docs/discovery/services.mdx b/website/content/docs/discovery/services.mdx deleted file mode 100644 index 9eba2e85bb5f..000000000000 --- a/website/content/docs/discovery/services.mdx +++ /dev/null @@ -1,701 +0,0 @@ ---- -layout: docs -page_title: Register Services with Service Definitions -description: >- - Define and register services and their health checks with Consul to make a service available for service discovery or service mesh access. Learn how to format service definitions with this reference page and sample code. ---- - -# Register Services with Service Definitions - -One of the main goals of service discovery is to provide a catalog of available -services. To that end, the agent provides a simple service definition format -to declare the availability of a service and to potentially associate it with -a health check. A health check associated with a service is considered to be an -application-level check. Define services in a configuration file or add it at -runtime using the HTTP interface. - -Complete the [Getting Started tutorials](/consul/tutorials/get-started-vms/virtual-machine-gs-service-discovery) to get hands-on experience registering a simple service with a health check on your local machine. - -## Service Definition - -Configure a service by providing the service definition to the agent. You can -either specify the configuration file using the `-config-file` option, or specify -the directory containing the service definition file with the `-config-dir` option. -Consul can load service definitions saved as `.json` or `.hcl` files. - -Send a `SIGHUP` to the running agent or use [`consul reload`](/consul/commands/reload) to check for new service definitions or to -update existing services. Alternatively, the service can be [registered dynamically](/consul/api-docs/agent/service#register-service) -using the [HTTP API](/consul/api-docs). - -A service definition contains a set of parameters that specify various aspects of the service, including how it is discovered by other services in the network. -All possible parameters are included in the following example, but only the top-level `service` parameter and its `name` parameter child are required by default. - - - -```hcl -service { - name = "redis" - id = "redis" - port = 80 - tags = ["primary"] - - meta = { - custom_meta_key = "custom_meta_value" - } - - tagged_addresses = { - lan = { - address = "192.168.0.55" - port = 8000 - } - - wan = { - address = "198.18.0.23" - port = 80 - } - } - - port = 8000 - socket_path = "/tmp/redis.sock" - enable_tag_override = false - - checks = [ - { - args = ["/usr/local/bin/check_redis.py"] - interval = "10s" - } - ] - - kind = "connect-proxy" - proxy_destination = "redis" - - proxy = { - destination_service_name = "redis" - destination_service_id = "redis1" - local_service_address = "127.0.0.1" - local_service_port = 9090 - local_service_socket_path = "/tmp/redis.sock" - mode = "transparent" - - transparent_proxy { - outbound_listener_port = 22500 - } - - mesh_gateway = { - mode = "local" - } - - expose = { - checks = true - - paths = [ - { - path = "/healthz" - local_path_port = 8080 - listener_port = 21500 - protocol = "http2" - } - ] - } - } - - connect = { - native = false - } - - weights = { - passing = 5 - warning = 1 - } - - token = "233b604b-b92e-48c8-a253-5f11514e4b50" - namespace = "foo" -} -``` - -```json -{ - "service": { - "id": "redis", - "name": "redis", - "tags": ["primary"], - "address": "", - "meta": { - "meta": "for my service" - }, - "tagged_addresses": { - "lan": { - "address": "192.168.0.55", - "port": 8000, - }, - "wan": { - "address": "198.18.0.23", - "port": 80 - } - }, - "port": 8000, - "socket_path": "/tmp/redis.sock", - "enable_tag_override": false, - "checks": [ - { - "args": ["/usr/local/bin/check_redis.py"], - "interval": "10s" - } - ], - "kind": "connect-proxy", - "proxy_destination": "redis", // Deprecated - "proxy": { - "destination_service_name": "redis", - "destination_service_id": "redis1", - "local_service_address": "127.0.0.1", - "local_service_port": 9090, - "local_service_socket_path": "/tmp/redis.sock", - "mode": "transparent", - "transparent_proxy": { - "outbound_listener_port": 22500 - }, - "config": {}, - "upstreams": [], - "mesh_gateway": { - "mode": "local" - }, - "expose": { - "checks": true, - "paths": [ - { - "path": "/healthz", - "local_path_port": 8080, - "listener_port": 21500, - "protocol": "http2" - } - ] - } - }, - "connect": { - "native": false, - "sidecar_service": {} - "proxy": { // Deprecated - "command": [], - "config": {} - } - }, - "weights": { - "passing": 5, - "warning": 1 - }, - "token": "233b604b-b92e-48c8-a253-5f11514e4b50", - "namespace": "foo" - } -} -``` - - - -The following table describes the available parameters for service definitions. - -### `service` - -This is the root-level parameter that defines the service. You can specify the parameters to configure the service. - -| Parameter | Description | Default | Required | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | ---------------------------- | -| `id` | String value that specifies the service ID.

    If not specified, the value of the `name` field will be used.

    Services must have unique IDs per node, so you should specify unique values if the default `name` will conflict with other services.

    | Value of the `name` parameter | Optional | -| `name` | Specifies the name of the service.
    The value for this parameter is used as the ID if the `id` parameter is not specified.
    We recommend using valid DNS labels for service definition names for compatibility with external DNSs. | None | Required | -| `tags` | List of string values that can be used to add service-level labels.
    For example, you can define tags that distinguish between `primary` and `secondary` nodes or service versions.
    We recommend using valid DNS labels for service definition IDs for compatibility with external DNSs.
    Tag values are opaque to Consul.
    | None | Optional | -| `address` | String value that specifies a service-specific IP address or hostname.
    If no value is specified, the IP address of the agent node is used by default.
    There is no service-side validation of this parameter. | IP address of the agent node | Optional | -| `meta` | Object that defines a map of the max 64 key/value pairs.
    The meta object has the same limitations as the node meta object in the node definition.
    Meta data can be retrieved per individual instance of the service. All instances of a given service have their own copy of the meta data.
    See [Adding Meta Data](#adding-meta-data) for supported parameters.
    | None | Optional | -| `tagged_addresses` | Tagged addresses are additional addresses that may be defined for a node or service. See [Tagged Addresses](#tagged-addresses) for details. | None | Optional | -| `port` | Integer value that specifies a service-specific port number. The port number should be specified when the `address` parameter is defined to improve service discoverability. | Optional | -| `socket_path` | String value that specifies the path to the service socket.
    Specify this parameter to expose the service to the mesh if the service listens on a Unix Domain socket. | None | Optional | -| `enable_tag_override` | Boolean value that determines if the anti-entropy feature for the service is enabled.
    If set to `true`, then external agents can update this service in the catalog and modify the tags.
    Subsequent local sync operations by this agent will ignore the updated tags.
    This parameter only applies to the locally-registered service. If multiple nodes register the same service, the `enable_tag_override` configuration, and all other service configuration items, operate independently.
    Updating the tags for services registered on one node is independent from the same service (by name) registered on another node.
    See [anti-entropy syncs](/consul/docs/architecture/anti-entropy) for additional information.
    | False | Optional | -| `checks` | Array of objects that define health checks for the service. See [Health Checks](#health-checks) for details. | None | Optional | -| `kind` | String value that identifies the service as a Connect proxy. See [Connect](#connect) for details. | None | Optional | -| `proxy_destination` | String value that specifies the _name_ of the destination service that the service currently being configured proxies to.
    This parameter is deprecated. Use `proxy.destination_service` instead.
    See [Connect](#connect) for additional information. | None | Optional | -| `proxy` | Object that defines the destination services that the service currently being configured proxies to. See [Proxy](#proxy) for additional information. | None | Optional | -| `connect` | Object that configures a Consul Connect service mesh connection. See [Connect](#connect) for details. | None | Optional | -| `weights` | Object that configures the weight of the service in terms of its DNS service (SRV) response. See [DNS SRV Weights](#dns-srv-weights) for details. | None | Optional | -| `token` | String value specifying the ACL token to be used to register the service (if the ACL system is enabled). The token is required for the service to interact with the service catalog. See [Security Configurations](#security-configurations) for details. | None | Required if ACLs are enabled | -| `namespace` | String value specifying the Consul Namespace where the service should be registered. See [Security Configurations](#security-configurations) for details. | None | Optional | - -### Adding Meta Data - -You can add semantic meta data to the service using the `meta` parameter. This parameter defines a map of max 64 key/value pairs. You can specify the following parameters to define meta data for the service. - -| Parameter | Description | Default | Required | -| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------- | -| `KEY` | String value that adds semantic metadata to the service.
    Keys can only have ASCII characters (`A` - `Z`, `a` - `z`, `0` - `9`, `_`, and `-`).
    Keys can not have special characters.
    Keys are limited to 128 characters.
    Values are limited to 512 characters. | None | Optional | - -### Security Configurations - -If the ACL system is enabled, specify a value for the `token` parameter to provide an ACL token. This token is -used for any interaction with the catalog for the service, including [anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - -Services registered in Consul clusters where both [Consul Namespaces](/consul/docs/enterprise/namespaces) -and the ACL system are enabled can be registered to specific namespaces that are associated with -ACL tokens scoped to the namespace. Services registered with a service definition -will not inherit the namespace associated with the ACL token specified in the `token` -field. The `namespace` _and_ the `token` parameters must be included in the service definition for the service to be registered to the -namespace that the ACL token is scoped to. - -### Health Checks - -You can add health checks to your service definition. Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database -to replace a failed secondary. The health check functionality is strongly integrated into the DNS interface, as well. If a service is failing its health check or a node has any failing system-level check, the DNS interface will omit that -node from any service query. - -The health check name is automatically generated as `service:`. If there are multiple service checks -registered, the ID will be generated as `service::` where -`` is an incrementing number starting from `1`. - -Consul includes several check types with different options. Refer to the [health checks documentation](/consul/docs/discovery/checks) for details. - -### Proxy - -Service definitions allow for an optional proxy registration. Proxies used with Connect -are registered as services in Consul's catalog. -See the [Proxy Service Registration](/consul/docs/connect/registration/service-registration) reference -for the available configuration options. - -### Connect - -The `kind` parameter determines the service's role. Services can be configured to perform several roles, but you must omit the `kind` parameter for typical non-proxy instances. - -The following roles are supported for service entries: - -- `connect-proxy`: Defines the configuration for a connect proxy -- `ingress-gateway`: Defines the configuration for an [ingress gateway](/consul/docs/connect/config-entries/ingress-gateway) -- `mesh-gateway`: Defines the configuration for a [mesh gateway](/consul/docs/connect/gateways/mesh-gateway) -- `terminating-gateway`: Defines the configuration for a [terminating gateway](/consul/docs/connect/config-entries/terminating-gateway#terminating-gateway) - -In the service definition example described above, the service is registered as a proxy because the `kind` property is set to `connect-proxy`. -The `proxy` parameter is also required for Connect proxy registrations and is only valid if `kind` is `connect-proxy`. -Refer to the [Proxy Service Registration](/consul/docs/connect/registration/service-registration) documentation for details about this type. - -When the `kind` parameter is set to `connect-proxy`, the only required parameter for the `proxy` configuration is `destination_service_name`. -Refer to the [complete proxy configuration example](/consul/docs/connect/registration/service-registration#complete-configuration-example) for additional information. - -The `connect` field can be specified to configure [Connect](/consul/docs/connect) for a service. This field is available in Consul 1.2.0 and later. The following parameters are available. - -| Parameter | Description | Default | Required | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------- | -| `native` | Boolean value that advertises the service as [Connect-native](/consul/docs/connect/native).
    If set to `true`, do not configure a `sidecar_service`. | `false` | Optional | -| `sidecar_service` | Object that defines a nested service definition.
    Do not configure if `native` is set to `true`. | See [Sidecar Service Registration](/consul/docs/connect/registration/sidecar-service) for default configurations. | Optional | - --> **Non-service registration roles**: The `kind` values supported for configuration entries are different than what is supported for service registrations. Refer to the [Configuration Entries](/consul/docs/connect/config-entries) documentation for information about non-service registration types. - -#### Deprecated parameters - -Different Consul Connect parameters are supported for different Consul versions. The following table describes changes applicable to service discovery. - -| Parameter | Description | Consul version | Status | -| ------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------- | --------------------------------------------------------------------------- | -| `proxy_destination` | Specified the proxy destination **in the root level** of the definition file. | 1.2.0 to 1.3.0 | Deprecated since 1.5.0.
    Use `proxy.destination_service_name` instead. | -| `connect.proxy` | Specified "managed" proxies, [which have been deprecated](/consul/docs/connect/proxies/managed-deprecated). | 1.2.0 (beta) to 1.3.0 (beta) | Deprecated. | - -### DNS SRV Weights - -You can configure how the service responds to DNS SRV requests by specifying a set of states/weights in the `weights` field. - -#### `weights` - -When DNS SRV requests are made, the response will include the weights specified for the given state of the service. -This allows some instances to be given higher weight if they have more capacity. It also allows load reduction on -services with checks in `warning` status by giving passing instances a higher weight. - -| Parameter | Description | Default | Required | -| --------- | --------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | -------- | -| `STATE` | Integer value indicating its weight. A higher number indicates more weight. | If not specified, the following weights are used:
    `"passing" : 1`
    `"warning" : 1` | Optional | - -If a service is `critical`, it is excluded from DNS responses. -Services with warning checks are included in responses by default, but excluded if the optional param `only_passing = true` -is present in the agent DNS configuration or the `passing` query parameter is used via the API. - -### Enable Tag Override and Anti-Entropy - -Services may also contain a `token` field to provide an ACL token. This token is -used for any interaction with the catalog for the service, including -[anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - -You can optionally disable the anti-entropy feature for this service using the -`enable_tag_override` flag. External agents can modify tags on services in the -catalog, so subsequent sync operations can either maintain tag modifications or -revert them. If `enable_tag_override` is set to `TRUE`, the next sync cycle may -revert some service properties, **but** the tags would maintain the updated value. -If `enable_tag_override` is set to `FALSE`, the next sync cycle will revert any -updated service properties, **including** tags, to their original value. - -It's important to note that this applies only to the locally registered -service. If you have multiple nodes all registering the same service -their `enable_tag_override` configuration and all other service -configuration items are independent of one another. Updating the tags -for the service registered on one node is independent of the same -service (by name) registered on another node. If `enable_tag_override` is -not specified the default value is false. See [anti-entropy -syncs](/consul/docs/architecture/anti-entropy) for more info. - -For Consul 0.9.3 and earlier you need to use `enableTagOverride`. Consul 1.0 -supports both `enable_tag_override` and `enableTagOverride` but the latter is -deprecated and has been removed as of Consul 1.1. - -### Tagged Addresses - -Tagged addresses are additional addresses that may be defined for a node or -service. Tagged addresses can be used by remote agents and services as alternative -addresses for communicating with the given node or service. Multiple tagged -addresses may be configured on a node or service. - -The following example describes the syntax for defining a tagged address. - - - -```hcl -service { - name = "redis" - port = 80 - tagged_addresses { - = { - address = "
    " - port = port - } - } -} -``` - -```json -{ - "service": { - "name": "redis", - "port": 80, - "tagged_addresses": { - "": { - "address": "
    ", - "port": port - } - } - } -} -``` - - - -The following table provides an overview of the various tagged address types supported by Consul. - -| Type | Description | Tags | -| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------- | -| LAN | LAN addresses are intended to be directly accessible only from services within the same Consul data center. See [LAN tags](#lan-tags) for details. | `lan`
    `lan_ipv4`
    `lan_ipv6` | -| Virtual | Virtual tagged addresses are logical address types that can be configured on [Connect](/consul/docs/connect)-enabled services. The virtual address provides a fixed IP address that can be used by downstream services when connecting to an upstream service. See [Virtual tags](#virtual-tags) for details. | `virtual` | -| WAN | Define a WAN address for the service or node when it should be accessed at an alternate address by services in a remote datacenter. See [WAN tags](#wan-tags) for details. | `wan`
    `wan_ipv4`
    `wan_ipv6` | - -#### LAN tags - -- `lan` - The IPv4 LAN address at which the node or service is accessible. -- `lan_ipv4` - The IPv4 LAN address at which the node or service is accessible. -- `lan_ipv6` - The IPv6 LAN address at which the node or service is accessible. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - lan = { - address = "192.0.2.10" - port = 80 - } - lan_ipv4 = { - address = "192.0.2.10" - port = 80 - } - lan_ipv6 = { - address = "2001:db8:1:2:cafe::1337" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "lan": { - "address": "192.0.2.10", - "port": 80 - }, - "lan_ipv4": { - "address": "192.0.2.10", - "port": 80 - }, - "lan_ipv6": { - "address": "2001:db8:1:2:cafe::1337", - "port": 80 - } - } - } -} -``` - - - - -#### Virtual tags - -Connections to virtual addresses are load balanced across available instances of a service, provided the following conditions are satisfied: - -1. [Transparent proxy](/consul/docs/connect/transparent-proxy) is enabled for the - downstream and upstream services. -1. The upstream service is not configured for individual instances to be - [dialed directly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly). - -Virtual addresses are not required to be routable IPs within the -network. They are strictly a control plane construct used to provide a fixed -address for the instances of a given logical service. Egress connections from -the proxy to an upstream service will be destined to the IP address of an -individual service instance, not the virtual address of the logical service. - -Use the following address tag to specify the logical address at which the -service can be reached by other services in the mesh. - -- `virtual` - The virtual IP address at which a logical service is reachable. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - virtual = { - address = "203.0.113.50" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "virtual": { - "address": "203.0.113.50", - "port": 80 - } - } - } -} -``` - - - - -#### WAN tags - -One or more of the following address tags can be configured for a node or service -to advertise how it should be accessed over the WAN. - -- `wan` - The IPv4 WAN address at which the node or service is accessible when - being dialed from a remote data center. -- `wan_ipv4` - The IPv4 WAN address at which the node or service is accessible - when being dialed from a remote data center. -- `wan_ipv6` - The IPv6 WAN address at which the node or service is accessible - when being dialed from a remote data center. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - wan = { - address = "198.51.100.200" - port = 80 - } - wan_ipv4 = { - address = "198.51.100.200" - port = 80 - } - wan_ipv6 = { - address = "2001:db8:5:6:1337::1eaf" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "wan": { - "address": "198.51.100.200", - "port": 80 - }, - "wan_ipv4": { - "address": "198.51.100.200", - "port": 80 - }, - "wan_ipv6": { - "address": "2001:db8:5:6:1337::1eaf", - "port": 80 - } - } - } -} -``` - - - - -## Multiple Service Definitions - -Multiple services definitions can be provided at once when registering services -via the agent configuration by using the plural `services` key (registering -multiple services in this manner is not supported using the HTTP API). - - - - - -```hcl -services { - id = "red0" - name = "redis" - tags = [ - "primary" - ] - address = "" - port = 6000 - checks = [ - { - args = ["/bin/check_redis", "-p", "6000"] - interval = "5s" - timeout = "20s" - } - ] -} -services { - id = "red1" - name = "redis" - tags = [ - "delayed", - "secondary" - ] - address = "" - port = 7000 - checks = [ - { - args = ["/bin/check_redis", "-p", "7000"] - interval = "30s" - timeout = "60s" - } - ] -} - -``` - - - - - -```json -{ - "services": [ - { - "id": "red0", - "name": "redis", - "tags": [ - "primary" - ], - "address": "", - "port": 6000, - "checks": [ - { - "args": ["/bin/check_redis", "-p", "6000"], - "interval": "5s", - "timeout": "20s" - } - ] - }, - { - "id": "red1", - "name": "redis", - "tags": [ - "delayed", - "secondary" - ], - "address": "", - "port": 7000, - "checks": [ - { - "args": ["/bin/check_redis", "-p", "7000"], - "interval": "30s", - "timeout": "60s" - } - ] - }, - ... - ] -} -``` - - - - -## Service and Tag Names with DNS - -Consul exposes service definitions and tags over the [DNS](/consul/docs/discovery/dns) -interface. DNS queries have a strict set of allowed characters and a -well-defined format that Consul cannot override. While it is possible to -register services or tags with names that don't match the conventions, those -services and tags will not be discoverable via the DNS interface. It is -recommended to always use DNS-compliant service and tag names. - -DNS-compliant service and tag names may contain any alpha-numeric characters, as -well as dashes. Dots are not supported because Consul internally uses them to -delimit service tags. - -## Service Definition Parameter Case - -For historical reasons Consul's API uses `CamelCased` parameter names in -responses, however its configuration file uses `snake_case` for both HCL and -JSON representations. For this reason the registration _HTTP APIs_ accept both -name styles for service definition parameters although APIs will return the -listings using `CamelCase`. - -Note though that **all config file formats require -`snake_case` fields**. We always document service definition examples using -`snake_case` and JSON since this format works in both config files and API -calls. diff --git a/website/content/docs/ecs/configuration-reference.mdx b/website/content/docs/ecs/configuration-reference.mdx index da0ee28f6813..fed887bc835d 100644 --- a/website/content/docs/ecs/configuration-reference.mdx +++ b/website/content/docs/ecs/configuration-reference.mdx @@ -185,7 +185,7 @@ Defines the Consul checks for the service. Each `check` object may contain the f | `method` | `string` | optional | Specifies the HTTP method to be used for an HTTP check. When no value is specified, `GET` is used. | | `name` | `string` | optional | The name of the check. | | `notes` | `string` | optional | Specifies arbitrary information for humans. This is not used by Consul internally. | -| `os_service` | `string` | optional | Specifies the name of a service on which to perform an [OS service check](/consul/docs/discovery/checks#osservice-check). The check runs according the frequency specified in the `interval` parameter. | +| `os_service` | `string` | optional | Specifies the name of a service on which to perform an [OS service check](/consul/docs/services/usage/checks#osservice-check). The check runs according the frequency specified in the `interval` parameter. | | `status` | `string` | optional | Specifies the initial status the health check. Must be one of `passing`, `warning`, `critical`, `maintenance`, or `null`. | | `successBeforePassing` | `integer` | optional | Specifies the number of consecutive successful results required before check status transitions to passing. | | `tcp` | `string` | optional | Specifies this is a TCP check. Must be an IP/hostname plus port to which a TCP connection is made every `interval`. | diff --git a/website/content/docs/ecs/manual/install.mdx b/website/content/docs/ecs/manual/install.mdx index e648c6cf3ea0..dddb9b310e50 100644 --- a/website/content/docs/ecs/manual/install.mdx +++ b/website/content/docs/ecs/manual/install.mdx @@ -373,11 +373,11 @@ configuration to a shared volume. ### `CONSUL_ECS_CONFIG_JSON` -Configuration is passed to the `consul-ecs` binary in JSON format using the `CONSUL_ECS_CONFIG_JSON` environment variable. +Consul uses the `CONSUL_ECS_CONFIG_JSON` environment variable to passed configurations to the `consul-ecs` binary in JSON format. -The following is an example of the configuration that might be used for a service named `example-client-app` with one upstream -service name `example-server-app`. The `proxy` and `service` blocks include information used by `consul-ecs-mesh-init` to perform -[service registration](/consul/docs/discovery/services) with Consul during task startup. The same configuration format is used for +The following example configures a service named `example-client-app` with one upstream +service name `example-server-app`. The `proxy` and `service` blocks include information used by `consul-ecs-mesh-init` to register the service with Consul during task start up. +The same configuration format is used for the `consul-ecs-health-sync` container. ```json @@ -409,7 +409,7 @@ the `consul-ecs-health-sync` container. | `proxy.upstreams` | list | The upstream services that your application calls over the service mesh, if any. The `destinationName` and `localBindPort` fields are required. | | `service.name` | string | The name used to register this service into the Consul service catalog. | | `service.port` | integer | The port your application listens on. Set to `0` if your application does not listen on any port. | -| `service.checks` | list | Consul [checks](/consul/docs/discovery/checks) to include so that Consul can run health checks against your application. | +| `service.checks` | list | Consul [checks](/consul/docs/services/usage/checks) to include so that Consul can run health checks against your application. | See the [Configuration Reference](/consul/docs/ecs/configuration-reference) for a complete reference of fields. diff --git a/website/content/docs/enterprise/admin-partitions.mdx b/website/content/docs/enterprise/admin-partitions.mdx index e5fdea32cfa7..e4a07104931a 100644 --- a/website/content/docs/enterprise/admin-partitions.mdx +++ b/website/content/docs/enterprise/admin-partitions.mdx @@ -61,7 +61,7 @@ The partition in which [`proxy-defaults`](/consul/docs/connect/config-entries/pr ### Cross-partition Networking -You can configure services to be discoverable by downstream services in any partition within the datacenter. Specify the upstream services that you want to be available for discovery by configuring the `exported-services` configuration entry in the partition where the services are registered. Refer to the [`exported-services` documentation](/consul/docs/connect/config-entries/exported-services) for details. Additionally, the requests made by downstream applications must have the correct DNS name for the Virtual IP Service lookup to occur. Service Virtual IP lookups allow for communications across Admin Partitions when using Transparent Proxy. Refer to the [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) for additional information. +You can configure services to be discoverable by downstream services in any partition within the datacenter. Specify the upstream services that you want to be available for discovery by configuring the `exported-services` configuration entry in the partition where the services are registered. Refer to the [`exported-services` documentation](/consul/docs/connect/config-entries/exported-services) for details. Additionally, the requests made by downstream applications must have the correct DNS name for the Virtual IP Service lookup to occur. Service Virtual IP lookups allow for communications across Admin Partitions when using Transparent Proxy. Refer to the [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups-for-consul-enterprise) for additional information. ### Cluster Peering diff --git a/website/content/docs/intro/index.mdx b/website/content/docs/intro/index.mdx index 50834137b4ed..90a1759fe233 100644 --- a/website/content/docs/intro/index.mdx +++ b/website/content/docs/intro/index.mdx @@ -24,7 +24,7 @@ Consul interacts with the _data plane_ through proxies. The data plane is the pa The core Consul workflow consists of the following stages: -- **Register**: Teams add services to the Consul catalog, which is a central registry that lets services automatically discover each other without requiring a human operator to modify application code, deploy additional load balancers, or hardcode IP addresses. It is the runtime source of truth for all services and their addresses. Teams can manually [define and register services](/consul/docs/discovery/services) using the CLI or the API, or you can automate the process in Kubernetes with [service sync](/consul/docs/k8s/service-sync). Services can also include health checks so that Consul can monitor for unhealthy services. +- **Register**: Teams add services to the Consul catalog, which is a central registry that lets services automatically discover each other without requiring a human operator to modify application code, deploy additional load balancers, or hardcode IP addresses. It is the runtime source of truth for all services and their addresses. Teams can manually [define](/consul/docs/services/usage/define-services) and [register](/consul/docs/services/usage/register-services-checks) using the CLI or the API, or you can automate the process in Kubernetes with [service sync](/consul/docs/k8s/service-sync). Services can also include health checks so that Consul can monitor for unhealthy services. - **Query**: Consul’s identity-based DNS lets you find healthy services in the Consul catalog. Services registered with Consul provide health information, access points, and other data that help you control the flow of data through your network. Your services only access other services through their local proxy according to the identity-based policies you define. - **Secure**: After services locate upstreams, Consul ensures that service-to-service communication is authenticated, authorized, and encrypted. Consul service mesh secures microservice architectures with mTLS and can allow or restrict access based on service identities, regardless of differences in compute environments and runtimes. diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx index 71e0d46638ee..3c85706cc626 100644 --- a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx +++ b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx @@ -319,7 +319,7 @@ Before you can call services from peered clusters, you must set service intentio -1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. +1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. diff --git a/website/content/docs/k8s/dns.mdx b/website/content/docs/k8s/dns.mdx index 5082919bd1d2..4262523a83d8 100644 --- a/website/content/docs/k8s/dns.mdx +++ b/website/content/docs/k8s/dns.mdx @@ -8,7 +8,7 @@ description: >- # Resolve Consul DNS Requests in Kubernetes One of the primary query interfaces to Consul is the -[DNS interface](/consul/docs/discovery/dns). You can configure Consul DNS in +[DNS interface](/consul/docs/services/discovery/dns-overview). You can configure Consul DNS in Kubernetes using a [stub-domain configuration](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configure-stub-domain-and-upstream-dns-servers) if using KubeDNS or a [proxy configuration](https://coredns.io/plugins/proxy/) if using CoreDNS. diff --git a/website/content/docs/k8s/service-sync.mdx b/website/content/docs/k8s/service-sync.mdx index 0cd6cb7c3822..db3e2bc9d833 100644 --- a/website/content/docs/k8s/service-sync.mdx +++ b/website/content/docs/k8s/service-sync.mdx @@ -20,7 +20,7 @@ automatically installed and configured using the Consul catalog enable Kubernetes services to be accessed by any node that is part of the Consul cluster, including other distinct Kubernetes clusters. For non-Kubernetes nodes, they can access services using the standard -[Consul DNS](/consul/docs/discovery/dns) or HTTP API. +[Consul DNS](/consul/docs/services/discovery/dns-overview) or HTTP API. **Why sync Consul services to Kubernetes?** Syncing Consul services to Kubernetes services enables non-Kubernetes services to be accessed using kube-dns and Kubernetes-specific diff --git a/website/content/docs/nia/configuration.mdx b/website/content/docs/nia/configuration.mdx index 4d4e8113f8e6..bd67ef5c07e4 100644 --- a/website/content/docs/nia/configuration.mdx +++ b/website/content/docs/nia/configuration.mdx @@ -197,7 +197,7 @@ Service registration requires that the [Consul token](/consul/docs/nia/configura | `default_check.enabled` | Optional | boolean | Enables CTS to create the default health check. | `true` | | `default_check.address` | Optional | string | The address to use for the default HTTP health check. Needs to include the scheme (`http`/`https`) and the port, if applicable. | `http://localhost:` or `https://localhost:`. Determined from the [port configuration](/consul/docs/nia/configuration#port) and whether [TLS is enabled](/consul/docs/nia/configuration#enabled-2) on the CTS API. | -The default health check is an [HTTP check](/consul/docs/discovery/checks#http-interval) that calls the [Health API](/consul/docs/nia/api/health). The following table describes the values CTS sets for this default check, corresponding to the [Consul register check API](/consul/api-docs/agent/check#register-check). If an option is not listed in this table, then CTS is using the default value. +The default health check is an [HTTP check](/consul/docs/services/usage/checks#http-checks) that calls the [Health API](/consul/docs/nia/api/health). The following table describes the values CTS sets for this default check, corresponding to the [Consul register check API](/consul/api-docs/agent/check#register-check). If an option is not listed in this table, then CTS is using the default value. | Parameter | Value | | --------- | ----- | diff --git a/website/content/docs/release-notes/consul/v1_13_x.mdx b/website/content/docs/release-notes/consul/v1_13_x.mdx index 736e84f9db5d..dd3f7cfe3090 100644 --- a/website/content/docs/release-notes/consul/v1_13_x.mdx +++ b/website/content/docs/release-notes/consul/v1_13_x.mdx @@ -15,7 +15,7 @@ description: >- - **Enables TLS on the Envoy Prometheus endpoint**: The Envoy prometheus endpoint can be enabled when `envoy_prometheus_bind_addr` is set and then secured over TLS using new CLI flags for the `consul connect envoy` command. These commands are: `-prometheus-ca-file`, `-prometheus-ca-path`, `-prometheus-cert-file` and `-prometheus-key-file`. The CA, cert, and key can be provided to Envoy by a Kubernetes mounted volume so that Envoy can watch the files and dynamically reload the certs when the volume is updated. -- **UDP Health Checks**: Adds the ability to register service discovery health checks that periodically send UDP datagrams to the specified IP/hostname and port. Refer to [UDP checks](/consul/docs/discovery/checks#udp-interval). +- **UDP Health Checks**: Adds the ability to register service discovery health checks that periodically send UDP datagrams to the specified IP/hostname and port. Refer to [UDP checks](/consul/docs//services/usage/checks#udp-checks). ## What's Changed diff --git a/website/content/docs/security/acl/acl-rules.mdx b/website/content/docs/security/acl/acl-rules.mdx index e9cd5c8b1287..3b63e327a20e 100644 --- a/website/content/docs/security/acl/acl-rules.mdx +++ b/website/content/docs/security/acl/acl-rules.mdx @@ -586,8 +586,7 @@ These actions may required an ACL token to complete. Use the following methods t This allows a single token to be used during all check registration operations. * Provide an ACL token with `service` and `check` definitions at registration time. This allows for greater flexibility and enables the use of multiple tokens on the same agent. - Refer to the [services](/consul/docs/discovery/services) and [checks](/consul/docs/discovery/checks) documentation for examples. - Tokens may also be passed to the [HTTP API](/consul/api-docs) for operations that require them. + Refer to the [services](/consul/docs/services/usage/define-services) and [checks](/consul/docs/services/usage/checks) documentation for examples. You can also pass tokens to the [HTTP API](/consul/api-docs) for operations that require them. ### Reading Imported Nodes @@ -815,12 +814,12 @@ to use for registration events: 2. Providing an ACL token with service and check definitions at registration time. This allows for greater flexibility and enables the use of multiple tokens on the same agent. Examples of what this looks like are available for - both [services](/consul/docs/discovery/services) and - [checks](/consul/docs/discovery/checks). Tokens may also be passed to the [HTTP - API](/consul/api-docs) for operations that require them. **Note:** all tokens + both [services](/consul/docs/services/usage/define-services) and + [checks](/consul/docs/services/usage/checks). Tokens may also be passed to the [HTTP + API](/consul/api-docs) for operations that require them. Note that all tokens passed to an agent are persisted on local disk to allow recovery from - restarts. See [`-data-dir` flag - documentation](/consul/docs/agent/config/cli-flags#_data_dir) for notes on securing + restarts. Refer to [`-data-dir` flag + documentation](/consul/docs/agent/config/cli-flags#_data_dir) for information about securing access. In addition to ACLs, in Consul 0.9.0 and later, the agent must be configured with diff --git a/website/content/docs/security/acl/acl-tokens.mdx b/website/content/docs/security/acl/acl-tokens.mdx index da14d52d88ae..068d8efa50e5 100644 --- a/website/content/docs/security/acl/acl-tokens.mdx +++ b/website/content/docs/security/acl/acl-tokens.mdx @@ -66,7 +66,7 @@ service = { -Refer to the [service definitions documentation](/consul/docs/discovery/services#service-definition) for additional information. +Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference) for additional information. ### Agent Requests diff --git a/website/content/docs/services/configuration/checks-configuration-reference.mdx b/website/content/docs/services/configuration/checks-configuration-reference.mdx new file mode 100644 index 000000000000..3159c977a208 --- /dev/null +++ b/website/content/docs/services/configuration/checks-configuration-reference.mdx @@ -0,0 +1,55 @@ +--- +layout: docs +page_title: Health Check Configuration Reference +description: -> + Use the health checks to direct safety functions, such as removing failing nodes and replacing secondary services. Learn how to configure health checks. +--- + +# Health Check Configuration Reference + +This topic provides configuration reference information for health checks. For information about the different kinds of health checks and guidance on defining them, refer to [Define Health Checks]. + +## Introduction +Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. You can configure health checks to monitor the health of the entire node. Refer to [Define Health Checks](/consul/docs/services/usage/checks) for information about how to define the differnet types of health checks available in Consul. + +## Check block +Specify health check options in the `check` block. To register two or more heath checks in the same configuration, use the [`checks` block](#checks-block). The following table describes the configuration options contained in the `check` block. + +| Parameter | Description | Check types | +| --- | --- | --- | +| `name` | Required string value that specifies the name of the check. Default is `service:`. If multiple service checks are registered, the autogenerated default is appended with colon and incrementing number starting with `1`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `id` | A unique string value that specifies an ID for the check. Default to the `name` value. If `name` values conflict, specify a unique ID to avoid overwriting existing checks with same ID on the same node. Consul auto-generates an ID if the check is defined in a service definition file. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `notes` | String value that provides a human-readabiole description of the check. The contents are not visible to Consul. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `interval` | Required string value that specifies how frequently to run the check. The `interval` parameter is required for supported check types. The value is parsed by the golang [time package formatting specification](https://golang.org/pkg/time/#ParseDuration). |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • Docker
  • gRPC
  • H2ping
    • | +| `timeout` | String value that specifies how long unsuccessful requests take to end with a timeout. The `timeout` is optional for the supported check types and has the following defaults:
  • Script: `30s`
  • HTTP: `10s`
  • TCP: `10s`
  • UDP: `10s`
  • gRPC: `10s`
  • H2ping: `10s`
    • |
  • Script
  • HTTP
  • TCP
  • UDP
  • gRPC
  • H2ping
    • | +| `status` | Optional string value that specifies the initial status of the health check. You can specify the following values:
  • `critical` (default)
  • `warning`
  • `passing`
    • |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `deregister_critical_service_after` | String value that specifies how long a service and its associated checks are allowed to be in a `critical` state. Consul deregisters services if they are `critical` for the specified amount of time. The value is parsed by the golang [time package formatting specification](https://golang.org/pkg/time/#ParseDuration) |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `success_before_passing` | Integer value that specifies how many consecutive times the check must pass before Consul marks the service or node as `passing`. Default is `0`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `failures_before_warning` | Integer value that specifies how many consecutive times the check must fail before Consul marks the service or node as `warning`. The value cannot be more than `failures_before_critical`. Defaults to the value specified for `failures_before_critical`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `failures_before_critical` | Integer value that specifies how many consecutive times the check must fail before Consul marks the service or node as `critical`. Default is `0`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `args` | Specifies a list of arguments strings to pass to the command line. The list of values includes the path to a script file or external application to invoke and any additional parameters for running the script or application. |
  • Script
  • Docker
    • | +| `docker_container_id` | Specifies the Docker container ID in which to run an external health check application. Specify the external application with the `args` parameter. |
  • Docker
    • | +| `shell` | String value that specifies the type of command line shell to use for running the health check application. Specify the external application with the `args` parameter. |
  • Docker
    • | +| `grpc` | String value that specifies the gRPC endpoint, including port number, to send requests to. Append the endpoint with `:/` and a service identifier to check a specific service. The endpoint must support the [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). |
  • gRPC
    • | +| `grpc_use_tls` | Boolean value that enables TLS for gRPC checks when set to `true`. |
  • gRPC
    • | +| `h2ping` | String value that specifies the HTTP2 endpoint, including port number, to send HTTP2 requests to. |
  • H2ping
    • | +| `h2ping_use_tls` | Boolean value that enables TLS for H2ping checks when set to `true`. |
  • H2ping
    • | +| `http` | String value that specifies an HTTP endpoint to send requests to. |
  • HTTP
    • | +| `tls_server_name` | String value that specifies the name of the TLS server that issues certificates. Defaults to the SNI determined by the address specified in the `http` field. Set the `tls_skip_verify` to `false` to disable this field. |
  • HTTP
    • | +| `tls_skip_verify` | Boolean value that disbles TLS for HTTP checks when set to `true`. Default is `false`. |
  • HTTP
    • | +| `method` | String value that specifies the request method to send during HTTP checks. Default is `GET`. |
  • HTTP
    • | +| `header` | Object that specifies header fields to send in HTTP check requests. Each header specified in `header` object contains a list of string values. |
  • HTTP
    • | +| `body` | String value that contains JSON attributes to send in HTTP check requests. You must escap the quotation marks around the keys and values for each attribute. |
  • HTTP
    • | +| `disable_redirects` | Boolean value that prevents HTTP checks from following redirects if set to `true`. Default is `false`. |
  • HTTP
    • | +| `os_service` | String value that specifies the name of the name of a service to check during an OSService check. |
  • OSService
    • | +| `service_id` | String value that specifies the ID of a service instance to associate with an OSService check. That service instance must be on the same node as the check. If not specified, the check verifies the health of the node. |
  • OSService
    • | +| `tcp` | String value that specifies an IP address or host and port number for the check establish a TCP connection with. |
  • TCP
    • | +| `udp` | String value that specifies an IP address or host and port number for the check to send UDP datagrams to. |
  • UDP
    • | +| `ttl` | String value that specifies how long to wait for an update from an external process during a TTL check. |
  • TTL
    • | +| `alias_service` | String value that specifies a service or node that the service associated with the health check aliases. |
  • Alias
    • | + + + +## Checks block +You can define multiple health checks in a single `checks` block. The `checks` block is an array of objects that contain the configuration options described in the [`check` block configuration reference](#check-block). + diff --git a/website/content/docs/services/configuration/services-configuration-overview.mdx b/website/content/docs/services/configuration/services-configuration-overview.mdx new file mode 100644 index 000000000000..55be8df32426 --- /dev/null +++ b/website/content/docs/services/configuration/services-configuration-overview.mdx @@ -0,0 +1,28 @@ +--- +layout: docs +page_title: Services Configuration Overview +description: -> + This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. +--- +# Services Configuration Overview + +This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. + +## Service definitions +A service definition contains a set of parameters that specify various aspects of the service, including how it is discovered by other services in the network. The service definition may also contain health check configurations. Refer to [Health Check Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for configuration details about health checks. + +Configure individual services and health checks by specifying parameters in the `service` block of a service definition file. Refer to [Define Services](/consul/docs/services/usage/define-services) for information about defining services. + +To register a service, provide the service definition to the Consul agent. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for information about registering services. + +Consul supports service definitions written in JSON and HCL. + +## Service defaults +Use the `service-defaults` configuration entry to define the default parameters for service definitions. This enables you to configure common settings, such as namespace or partition for Consul Enterprise deployments, in a single definition. + +You can use `service-defaults` configuration entries on virtual machines and in Kubernetes environments. + +## ACLs +Services registered in Consul clusters where both Consul Namespaces and the ACL system are enabled can be registered to specific namespaces that are associated with ACL tokens scoped to the namespace. Services registered with a service definition will not inherit the namespace associated with the ACL token specified in the token field. The namespace and the token parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + + diff --git a/website/content/docs/services/configuration/services-configuration-reference.mdx b/website/content/docs/services/configuration/services-configuration-reference.mdx new file mode 100644 index 000000000000..d3ee69e0373f --- /dev/null +++ b/website/content/docs/services/configuration/services-configuration-reference.mdx @@ -0,0 +1,654 @@ +--- +page_title: Service Configuration Reference +description: Use the service definition to configure and register services to the Consul catalog, including services used as proxies in a Consul service mesh +--- + +# Services Configuration Reference + +This topic describes the options you can use to define services for registering them with Consul. Refer to the following topics for usage information: + +- [Define Services](/consul/docs/services/usage/define-services) +- [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) + +## Configuration model +The following outline shows how to format the configurations in the root `service` block. Click on a property name to view details about the configuration. + +- [`name`](#name): string | required +- [`id`](#id): string | optional +- [`address`](#address): string | optional +- [`port`](#port): integer | optional +- [`tags`](#tags): list of strings | optional +- [`meta`](#meta): object | optional + - [_`custom_meta_key`_](#meta): string | optional +- [`tagged_addresses`](#tagged_addresses): object | optional + - [`lan`](#tagged_addresses-lan): object | optional + - [`address`](#tagged_addresses-lan): string | optional + - [`port`](#tagged_addresses-lan): integer | optional + - [`wan`](#tagged_addresses-wan): object | optional + - [`address`](#tagged_addresses-wan): string | optional + - [`port`](#tagged_addresses-wan): integer | optional +- [`socket_path`](#socket_path): string | optional +- [`enable_tag_override`](#enable_tag_override): boolean | optional +- [`checks`](#checks) : list of objects | optional +- [`kind`](#kind): string | optional +- [`proxy`](#proxy): object | optional +- [`connect`](#connect): object | optional + - [`native`](#connect): boolean | optional + - [`sidecar_service`](#connect): object | optional +- [`weights`](#weights): object | optional + - [`passing`](#weights): integer | optional + - [`warning`](#weights): integer | optional +- [`token`](#token): string | required if ACLs are enabled +- [`namespace`](#namespace): string | optional | + +## Specification +This topic provides details about the configuration parameters. + +### name +Required value that specifies a name for the service. We recommend using valid DNS labels for service definition names for compatibility with external DNSs. The value for this parameter is used as the ID if the `id` parameter is not specified. + +- Type: string +- Default: none + +### id +Specifies an ID for the service. Services on the same node must have unique IDs. We recommend specifying unique values if the default name conflicts with other services. + +- Type: string +- Default: Value of the `name` field. + +### address +String value that specifies a service-specific IP address or hostname. +If no value is specified, the IP address of the agent node is used by default. +There is no service-side validation of this parameter. + +- Type: string +- Default: IP address of the agent node + +### port +Specifies a port number for the service. To improve service discoverability, we recommend specifying the port number, as well as an address in the [`tagged_addresses`](#tagged_addresses) parameter. + +- Type: integer +- Default: Port number of the agent + +### tags +Specifies a list of string values that add service-level labels. Tag values are opaque to Consul. We recommend using valid DNS labels for service definition IDs for compatibility with external DNSs. In the following example, the service is tagged as `v2` and `primary`: + +```hcl +tags = ["v2", "primary"] +``` + +Consul uses tags as an anti-entropy mechanism to maintain the state of the cluster. You can disable the anti-entropy feature for a service using the [`enable_tag_override`](#enable_tag_override) setting, which enables external agents to modify tags on services in the catalog. Refer to [Modify anti-entropy synchronization](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional usage information. + +### meta +The `meta` field contains custom key-value pairs that associate semantic metadata with the service. You can specify up to 64 pairs that meet the following requirements: + +- Keys and values must be strings. +- Keys can only contain ASCII characters (`A` -` Z`, `a`- `z`, `0` - `9`, `_`, and `-`). +- Keys can not have special characters. +- Keys are limited to 128 characters. +- Values are limited to 512 characters. + +In the following example, the `env` key is set to `prod`: + + + +```hcl +meta = { + env = "prod" +} +``` + +```json +"meta" : { + "env" : "prod" +} +``` + + + +### tagged_addresses +The `tagged_address` field is an object that configures additional addresses for a node or service. Remote agents and services can communicate with the service using a tagged address as an alternative to the address specified in the [`address`](#address) field. You can configure multiple addresses for a node or service. The following tags are supported: + +- [`lan`](#tagged_addresses-lan): IPv4 LAN address where the node or service is accessible. +- [`lan_ipv4`](#tagged_addresses-lan): IPv4 LAN address where the node or service is accessible. +- [`lan_ipv6`](#tagged_addresses-lan): IPv6 LAN address where the node or service is accessible. +- [`virtual`](#tagged_addresses-virtual): A fixed address for the instances of a given logical service. +- [`wan`](#tagged_addresses-wan): IPv4 WAN address where the node or service is accessible when dialed from a remote data center. +- [`wan_ipv4`](#tagged_addresses-wan): IPv4 WAN address where the node or service is accessible when dialed from a remote data center. +- [`wan_ipv6`](#tagged_addresses-lan): IPv6 WAN address at which the node or service is accessible when being dialed from a remote data center. + +### tagged_addresses.lan +Object that specifies either an IPv4 or IPv6 LAN address and port number where the service or node is accessible. You can specify one or more of the following fields: + +- `lan` +- `lan_ipv4` +- `lan_ipv6` + +The field contains the following parameters: + +- `address` +- `port` + +In the following example, the `redis` service has an IPv4 LAN address of `192.0.2.10:80` and IPv6 LAN address of `[2001:db8:1:2:cafe::1337]:80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + lan = { + address = "192.0.2.10" + port = 80 + } + lan_ipv4 = { + address = "192.0.2.10" + port = 80 + } + lan_ipv6 = { + address = "2001:db8:1:2:cafe::1337" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "lan": { + "address": "192.0.2.10", + "port": 80 + }, + "lan_ipv4": { + "address": "192.0.2.10", + "port": 80 + }, + "lan_ipv6": { + "address": "2001:db8:1:2:cafe::1337", + "port": 80 + } + } + } +} +``` + + + +### tagged_addresses.virtual +Object that specifies a fixed IP address and port number that downstream services in a service mesh can use to connect to the service. The `virtual` field contains the following parameters: + +- `address` +- `port` + +Virtual addresses are not required to be routable IPs within the network. They are strictly a control plane construct used to provide a fixed address for the instances of a logical service. Egress connections from the proxy to an upstream service go to the IP address of an individual service instance and not the virtual address of the logical service. + +If the following conditions are met, connections to virtual addresses are load balanced across available instances of a service, provided the following conditions are satisfied: + +1. [Transparent proxy](/consul/docs/connect/transparent-proxy) is enabled for the downstream and upstream services. +1. The upstream service is not configured for individual instances to be [dialed directly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly). + +In the following example, the downstream services in the mesh can connect to the `redis` service at `203.0.113.50` on port `80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + virtual = { + address = "203.0.113.50" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "virtual": { + "address": "203.0.113.50", + "port": 80 + } + } + } +} +``` + + + +### tagged_addresses.wan +Object that specifies either an IPv4 or IPv6 WAN address and port number where the service or node is accessible from a remote datacenter. You can specify one or more of the following fields: + +- `wan` +- `wan_ipv4` +- `wan_ipv6` + +The field contains the following parameters: + +- `address` +- `port` + +In the following example, services or nodes in remote datacenters can reach the `redis` service at `198.51.100.200:80` and `[2001:db8:5:6:1337::1eaf]:80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + wan = { + address = "198.51.100.200" + port = 80 + } + wan_ipv4 = { + address = "198.51.100.200" + port = 80 + } + wan_ipv6 = { + address = "2001:db8:5:6:1337::1eaf" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "wan": { + "address": "198.51.100.200", + "port": 80 + }, + "wan_ipv4": { + "address": "198.51.100.200", + "port": 80 + }, + "wan_ipv6": { + "address": "2001:db8:5:6:1337::1eaf", + "port": 80 + } + } + } +} +``` + + + +### socket_path +String value that specifies the path to the service socket. Specify this parameter to expose the service to the mesh if the service listens on a Unix Domain socket. + +- Type: string +- Default: none + +### enable_tag_override +Boolean value that determines if the anti-entropy feature for the service is enabled. +Set to `true` to allow external Consul agents modify tags on the services in the Consul catalog. The local Consul agent ignores updated tags during subsequent sync operations. + +This parameter only applies to the locally-registered service. If multiple nodes register a service with the same `name`, the `enable_tag_override` configuration, and all other service configuration items, operate independently. + +Refer to [Modify anti-entropy synchronization](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional usage information. + +- Type: boolean +- Default: `false` + +### checks +The `checks` block contains an array of objects that define health checks for the service. Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. Refer to [Health Check Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about configuring health checks. + +### kind +String value that identifies the service as a proxy and determines its role in the service mesh. Do not configure the `kind` parameter for non-proxy service instances. Refer to [Consul Service Mesh](/consul/docs/connect) for additional information. + +You can specify the following values: + +- `connect-proxy`: Defines the configuration for a service mesh proxy. Refer to [Register a Service Mesh Proxy Outside of a Service Registration](/consul/docs/connect/registration/service-registration) for details about registering a service as a service mesh proxy. +- `ingress-gateway`: Defines the configuration for an [ingress gateway](/consul/docs/connect/config-entries/ingress-gateway) +- `mesh-gateway`: Defines the configuration for a [mesh gateway](/consul/docs/connect/gateways/mesh-gateway) +- `terminating-gateway`: Defines the configuration for a [terminating gateway](/consul/docs/connect/gateways/terminating-gateway) + +For non-service registration roles, the `kind` field has a different context when used to define configuration entries, such as `service-defaults`. Refer to the documentation for the configuration entry you want to implement for additional information. + +### proxy +Object that specifies proxy configurations when the service is configured to operate as a proxy in a service mesh. Do not configure the `proxy` parameter for non-proxy service instances. Refer to [Register a Service Mesh Proxy Outside of a Service Registration](/consul/docs/connect/registration/service-registration) for details about registering your service as a service mesh proxy. Refer to [`kind`](#kind) for information about the types of proxies you can define. Services that you assign proxy roles to are registered as services in the catalog. + +### connect +Object that configures a Consul service mesh connection. You should only configure the `connect` block of parameters if you are using Consul service mesh. Refer to [Consul Service Mesh](/consul/docs/connect) for additional information. + +The following table describes the parameters that you can place in the `connect` block: + +| Parameter | Description | Default | +| --- | --- | --- | +| `native` | Boolean value that advertises the service as a native service mesh proxy. Use this parameter to integrate your application with the `connect` API. Refer to [Service Mesh Native App Integration Overview](/consul/docs/connect/native) for additional information. If set to `true`, do not configure a `sidecar_service`. | `false` | +| `sidecar_service` | Object that defines a sidecar proxy for the service. Do not configure if `native` is set to `true`. Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for usage and configuration details. | Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for default configurations. | + +### weights +Object that configures how the service responds to DNS SRV requests based on the service's health status. Configuring allows service instances with more capacity to respond to DNS SRV requests. It also reduces the load on services with checks in `warning` status by giving passing instances a higher weight. + +You can specify one or more of the following states and configure an integer value indicating its weight: + +- `passing` +- `warning` +- `critical` + +Larger integer values increase the weight state. Services have the following default weights: + +- `"passing" : 1` +- `"warning" : 1` + +Services in a `critical` state are excluded from DNS responses by default. Services with `warning` checks are included in responses by default. Refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups) for additional information. + +In the following example, service instances in a `passing` state respond to DNS SRV requests, while instances in a `critical` instance can still respond at a lower frequency: + + + +```hcl +service { + ## ... + weights = { + passing = 3 + warning = 2 + critical = 1 + } + ## ... +} +``` + +```json +"service": { + ## ... + "weights": { + "passing": 3, + "warning": 2, + "critical": 1 + }, + ## ... +} +``` + + + +### token +String value that specifies the ACL token to present when registering the service if ACLs are enabled. The token is required for the service to interact with the service catalog. + +If [ACLs](/consul/docs/security/acl) and [namespaces](/consul/docs/enterprise/namespaces) are enabled, you can register services scoped to the specific [`namespace`](#namespace) associated with the ACL token in a Consul cluster. + +Services registered with a service definition do not inherit the namespace associated with the ACL token specified in the token field. The `namespace` and `token` parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + +- Type: string +- Default: none + +### namespace +String value that specifies the namespace in which to register the service. Refer [Namespaces](/consul/docs/enterprise/namespaces) for additional information. + +- Type: string +- Default: none + +## Multiple service definitions + +You can define multiple services in a single definition file in the `servcies` block. This enables you register multiple services in a single command. Note that the HTTP API does not support the `services` block. + + + +```hcl +services { + id = "red0" + name = "redis" + tags = [ + "primary" + ] + address = "" + port = 6000 + checks = [ + { + args = ["/bin/check_redis", "-p", "6000"] + interval = "5s" + timeout = "20s" + } + ] +} +services { + id = "red1" + name = "redis" + tags = [ + "delayed", + "secondary" + ] + address = "" + port = 7000 + checks = [ + { + args = ["/bin/check_redis", "-p", "7000"] + interval = "30s" + timeout = "60s" + } + ] +} +``` + +```json +{ + "services": [ + { + "id": "red0", + "name": "redis", + "tags": [ + "primary" + ], + "address": "", + "port": 6000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "6000"], + "interval": "5s", + "timeout": "20s" + } + ] + }, + { + "id": "red1", + "name": "redis", + "tags": [ + "delayed", + "secondary" + ], + "address": "", + "port": 7000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "7000"], + "interval": "30s", + "timeout": "60s" + } + ] + } + ] +} +``` + + + +## Example definition +The following example includes all possible parameters, but only the top-level `service` parameter and its `name` parameter are required by default. + + + +```hcl +service { + name = "redis" + id = "redis" + port = 80 + tags = ["primary"] + + meta = { + custom_meta_key = "custom_meta_value" + } + + tagged_addresses = { + lan = { + address = "192.168.0.55" + port = 8000 + } + + wan = { + address = "198.18.0.23" + port = 80 + } + } + + port = 8000 + socket_path = "/tmp/redis.sock" + enable_tag_override = false + + checks = [ + { + args = ["/usr/local/bin/check_redis.py"] + interval = "10s" + } + ] + + kind = "connect-proxy" + proxy_destination = "redis" + + proxy = { + destination_service_name = "redis" + destination_service_id = "redis1" + local_service_address = "127.0.0.1" + local_service_port = 9090 + local_service_socket_path = "/tmp/redis.sock" + mode = "transparent" + + transparent_proxy { + outbound_listener_port = 22500 + } + + mesh_gateway = { + mode = "local" + } + + expose = { + checks = true + + paths = [ + { + path = "/healthz" + local_path_port = 8080 + listener_port = 21500 + protocol = "http2" + } + ] + } + } + + connect = { + native = false + } + + weights = { + passing = 5 + warning = 1 + } + + token = "233b604b-b92e-48c8-a253-5f11514e4b50" + namespace = "foo" +} +``` + +```json +{ + "service": { + "id": "redis", + "name": "redis", + "tags": ["primary"], + "address": "", + "meta": { + "meta": "for my service" + }, + "tagged_addresses": { + "lan": { + "address": "192.168.0.55", + "port": 8000, + }, + "wan": { + "address": "198.18.0.23", + "port": 80 + } + }, + "port": 8000, + "socket_path": "/tmp/redis.sock", + "enable_tag_override": false, + "checks": [ + { + "args": ["/usr/local/bin/check_redis.py"], + "interval": "10s" + } + ], + "kind": "connect-proxy", + "proxy_destination": "redis", // Deprecated + "proxy": { + "destination_service_name": "redis", + "destination_service_id": "redis1", + "local_service_address": "127.0.0.1", + "local_service_port": 9090, + "local_service_socket_path": "/tmp/redis.sock", + "mode": "transparent", + "transparent_proxy": { + "outbound_listener_port": 22500 + }, + "config": {}, + "upstreams": [], + "mesh_gateway": { + "mode": "local" + }, + "expose": { + "checks": true, + "paths": [ + { + "path": "/healthz", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http2" + } + ] + } + }, + "connect": { + "native": false, + "sidecar_service": {} + "proxy": { // Deprecated + "command": [], + "config": {} + } + }, + "weights": { + "passing": 5, + "warning": 1 + }, + "token": "233b604b-b92e-48c8-a253-5f11514e4b50", + "namespace": "foo" + } +} +``` + + + + + + diff --git a/website/content/docs/services/discovery/dns-configuration.mdx b/website/content/docs/services/discovery/dns-configuration.mdx new file mode 100644 index 000000000000..7e43e7b75bb2 --- /dev/null +++ b/website/content/docs/services/discovery/dns-configuration.mdx @@ -0,0 +1,76 @@ +--- +layout: docs +page_title: Configure Consul DNS Behavior +description: -> + Learn how to modify the default DNS behavior so that services and nodes can easily discover other services and nodes in your network. +--- + +# Configure Consul DNS Behavior + +This topic describes the default behavior of the Consul DNS functionality and how to customize how Consul performs queries. + +## Introduction +The Consul DNS is the primary interface for querying records when Consul service mesh is disabled and your network runs in a non-Kubernetes environment. The DNS enables you to look up services and nodes registered with Consul using terminal commands instead of making HTTP API requests to Consul. Refer to the [Discover Consul Nodes and Services Overview](/consul/docs/services/discovery/dns-overview) for additional information. + +## Configure DNS behaviors +By default, the Consul DNS listens for queries at `127.0.0.1:8600` and uses the `consul` domain. Specify the following parameters in the agent configuration to determine DNS behavior when querying services: + +- [`client_addr`](/consul/docs/agent/config/config-files#client_addr) +- [`ports.dns`](/consul/docs/agent/config/config-files#dns_port) +- [`recursors`](/consul/docs/agent/config/config-files#recursors) +- [`domain`](/consul/docs/agent/config/config-files#domain) +- [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) +- [`dns_config`](/consul/docs/agent/config/config-files#dns_config) + +### Configure WAN address translation +By default, Consul DNS queries return a node's local address, even when being queried from a remote datacenter. You can configure the DNS to reach a node from outside its datacenter by specifying the address in the following configuration fields in the Consul agent: + +- [advertise-wan](/consul/docs/agent/config/cli-flags#_advertise-wan) +- [translate_wan_addrs](/consul//docs/agent/config/config-files#translate_wan_addrs) + +### Use a custom DNS resolver library +You can specify a list of addresses in the agent's [`recursors`](/consul/docs/agent/config/config-files#recursors) field to provide upstream DNS servers that recursively resolve queries that are outside the service domain for Consul. + +Nodes that query records outside the `consul.` domain resolve to an upstream DNS. You can specify IP addresses or use `go-sockaddr` templates. Consul resolves IP addresses in the specified order and ignores duplicates. + +### Enable non-Consul queries +You enable non-Consul queries to be resolved by setting Consul as the DNS server for a node and providing a [`recursors`](/consul/docs/agent/config/config-files#recursors) configuration. + +### Forward queries to an agent +You can forward all queries sent to the `consul.` domain from the existing DNS server to a Consul agent. Refer to [Forward DNS for Consul Service Discovery](/consul/tutorials/networking/dns-forwarding) for instructions. + +### Query an alternate domain +By default, Consul responds to DNS queries in the `consul` domain, but you can set a specific domain for responding to DNS queries by configuring the [`domain`](/consul/docs/agent/config/config-files#domain) parameter. + +You can also specify an additional domain in the [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) agent configuration option, which configures Consul to respond to queries in a secondary domain. Configuring an alternate domain may be useful during a DNS migration or to distinguish between internal and external queries, for example. + +Consul's DNS response uses the same domain as the query. + +In the following example, the `alt_domain` parameter in the agent configuration is set to `test-domain`, which enables operators to query the domain: + +```shell-session +$ dig @127.0.0.1 -p 8600 consul.service.test-domain SRV + +;; QUESTION SECTION: +;consul.service.test-domain. IN SRV + +;; ANSWER SECTION: +consul.service.test-domain. 0 IN SRV 1 1 8300 machine.node.dc1.test-domain. + +;; ADDITIONAL SECTION: +machine.node.dc1.test-domain. 0 IN A 127.0.0.1 +machine.node.dc1.test-domain. 0 IN TXT "consul-network-segment=" +``` +#### PTR queries +Responses to pointer record (PTR) queries, such as `.in-addr.arpa.`, always use the [primary domain](/consul/docs/agent/config/config-files#domain) and not the alternative domain. + +### Caching +By default, DNS results served by Consul are not cached. Refer to the [DNS Caching tutorial](/consul/tutorials/networking/dns-caching) for instructions on how to enable caching. + + + + + + + + diff --git a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx new file mode 100644 index 000000000000..d19f41c9ea67 --- /dev/null +++ b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx @@ -0,0 +1,100 @@ +--- +layout: docs +page_title: Enable Dynamic DNS Queries +description: -> + Learn how to dynamically query the Consul DNS using prepared queries, which enable robust service and node lookups. +--- + +# Enable Dynamic DNS Queries +This topic describes how to dynamically query the Consul catalog using prepared queries. Prepared queries are configurations that enable you to register a complex service query and execute it on demand. For information about how to perform standard node and service lookups, refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups). + +## Introduction +Prepared queries provide a rich set of lookup features, such as filtering by multiple tags and automatically failing over to look for services in remote datacenters if no healthy nodes are available in the local datacenter. You can also create prepared query templates that match names using a prefix match, allowing a single template to apply to potentially many services. Refer to [Query Consul Nodes and Services Overview](/consul/docs/services/discovery/dns-overview) for additional information about DNS query behaviors. + +## Requirements +Consul 0.6.4 or later is required to create prepared query templates. + +### ACLs +If ACLs are enabled, the querying service must present a token linked to permissions that enable read access for query, service, and node resources. Refer to the following documentation for information about creating policies to enable read access to the necessary resources: + +- [Prepared query rules](/consul/docs/security/acl/acl-rules#prepared-query-rules) +- [Service rules](/consul/docs/security/acl/acl-rules#service-rules) +- [Node rules](/consul/docs/security/acl/acl-rules#node-rules) + +## Create prepared queries +Refer to the [prepared query reference](/consul/api-docs/query#create-prepared-query) for usage information. + +1. Specify the prepared query options in JSON format. The following prepared query targets all instances of the `redis` service in `dc1` and `dc2`: + + ```json + { + "Name": "my-query", + "Session": "adf4238a-882b-9ddc-4a9d-5b6758e4159e", + "Token": "", + "Service": { + "Service": "redis", + "Failover": { + "NearestN": 3, + "Datacenters": ["dc1", "dc2"] + }, + "Near": "node1", + "OnlyPassing": false, + "Tags": ["primary", "!experimental"], + "NodeMeta": { + "instance_type": "m3.large" + }, + "ServiceMeta": { + "environment": "production" + } + }, + "DNS": { + "TTL": "10s" + } + } + ``` + + Refer to the [prepared query configuration reference](/consul/api-docs/query#create-prepared-query) for information about all available options. + +1. Send the query in a POST request to the [`/query` API endpoint](/consul/api-docs/query). If the request is successful, Consul prints an ID for the prepared query. + + In the following example, the prepared query configuration is stored in the `payload.json` file: + + ```shell-session + $ curl --request POST --data @payload.json http://127.0.0.1:8500/v1/query + {"ID":"014af5ff-29e6-e972-dcf8-6ee602137127"}% + ``` +1. To run the query, send a GET request to the endpoint and specify the ID returned from the POST call. + + ```shell-session + $ curl http://127.0.0.1:8500/v1/query/14af5ff-29e6-e972-dcf8-6ee602137127/execute\?near\=_agent + ``` + +## Execute prepared queries +You can execute a prepared query using the standard lookup format or the strict RFC 2782 SRV lookup. + +### Standard lookup + +Use the following format to execute a prepared query using the standard lookup format: + +``` +.query[.]. +``` + +Refer [Standard lookups](/consul/docs/services/discovery/dns-static-lookups#standard-lookups) for additional information about the standard lookup format in Consul. + +### RFC 2782 SRV lookup +Use the following format to execute a prepared query using the RFC 2782 lookup format: + +``` +_._tcp.query[.]. +``` + +For additional information about following the RFC 2782 SRV lookup format in Consul, refer to [RFC 2782 Lookup](/consul/docs/services/discovery/dns-static-lookups#rfc-2782-lookup). For general information about the RFC 2782 specification, refer to [A DNS RR for specifying the location of services \(DNS SRV\)](https://tools.ietf.org/html/rfc2782). + +### Lookup options +The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of this Consul agent. + +The `query name` or `id` subdomain is the name or ID of an existing prepared query. + +## Results +To allow for simple load balancing, Consul returns the set of nodes in random order for each query. Prepared queries support A and SRV records. SRV records provide the port that a service is registered. Consul only serves SRV records if the client specifically requests them. \ No newline at end of file diff --git a/website/content/docs/services/discovery/dns-overview.mdx b/website/content/docs/services/discovery/dns-overview.mdx new file mode 100644 index 000000000000..53baac080e77 --- /dev/null +++ b/website/content/docs/services/discovery/dns-overview.mdx @@ -0,0 +1,41 @@ +--- +layout: docs +page_title: DNS Usage Overview +description: >- + For service discovery use cases, Domain Name Service (DNS) is the main interface to look up, query, and address Consul nodes and services. Learn how a Consul DNS lookup can help you find services by tag, name, namespace, partition, datacenter, or domain. +--- + +# DNS Usage Overview + +This topic provides overview information about how to look up Consul nodes and services using the Consul DNS. + +## Consul DNS +The Consul DNS is the primary interface for discovering services registered in the Consul catalog. The DNS enables you to look up services and nodes registered with Consul using terminal commands instead of making HTTP API requests to Consul. + +We recommend using the DNS for service discovery in virtual machine (VM) environments because it removes the need to modify native applications so that they can consume the Consul service discovery APIs. + +The DNS has several default configurations, but you can customize how the server processes lookups. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for additional information. + +### DNS versus native app integration +You can use DNS to reach services registered with Consul or modify your application to natively consume the Consul service discovery HTTP APIs. + +We recommend using the DNS because it is less invasive. You do not have to modify your application with Consul to retrieve the service’s connection information. Instead, you can use a DNS fully qualified domain (FQDN) that conforms to Consul's lookup format to retreive the relevant information. + +Refer to [ Native App Integration](/consul/docs/connect/native) and its [Go package](/consul/docs/connect/native/go) for additional information. + +### DNS versus upstreams +If you are using Consul for service discovery and have not enabled service mesh features, then use the DNS to discover services and nodes in the Consul catalog. + +If you are using Consul for service mesh on VMs, you can use upstreams or DNS. We recommend using upstreams because you can query services and nodes without modifying the application code or environment variables. Refer to [Upstream Configuration Reference](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) for additional information. + +If you are using Consul on Kubernetes, refer to [the upstreams annotation documentation](/consul/docs/k8s/annotations-and-labels#consul-hashicorp-com-connect-service-upstreams) for additional information. + +## Static queries +Node lookups and service lookups are the fundamental types of static queries. Depending on your use case, you may need to use different query methods and syntaxes to query the DNS for services and nodes. + +Consul relies on a very specific format for queries to resolve names. Note that all queries are case-sensitive. + +Refer to [Perform Static DNS Lookups](/consul/docs/services/discovery/dns-static-lookups) for details about how to perform node and service lookups. + +## Prepared queries +Prepared queries are configurations that enable you to register complex DNS queries. They provide lookup features that extend Consul's service discovery capabilities, such as filtering by multiple tags and automatically querying remote datacenters for services if no healthy nodes are available in the local datacenter. You can also create prepared query templates that match names using a prefix match, allowing a single template to apply to potentially many services. Refer to [Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for additional information. diff --git a/website/content/docs/services/discovery/dns-static-lookups.mdx b/website/content/docs/services/discovery/dns-static-lookups.mdx new file mode 100644 index 000000000000..d95987671ef9 --- /dev/null +++ b/website/content/docs/services/discovery/dns-static-lookups.mdx @@ -0,0 +1,357 @@ +--- +layout: docs +page_title: Perform Static DNS Queries +description: -> + Learn how to use standard Consul DNS lookup formats to enable service discovery for services and nodes. +--- + +# Perform Static DNS Queries +This topic describes how to query the Consul DNS to look up nodes and services registered with Consul. Refer to [Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for information about using prepared queries. + +## Introduction +Node lookups and service lookups are the fundamental types of queries you can perform using the Consul DNS. Node lookups interrogate the catalog for named Consul agents. Service lookups interrogate the catalog for services registered with Consul. Refer to [DNS Usage Overivew](/consul/docs/services/discovery/dns-overview) for additional background information. + +## Requirements +All versions of Consul support DNS lookup features. + +### ACLs +If ACLs are enabled, you must present a token linked with the necessary policies. We recommend using a separate token in production deployments for querying the DNS. By default, Consul agents resolve DNS requests using the preconfigured tokens in order of precedence: + +The agent's [`default` token](/consul/docs/agent/config/config-files#acl_tokens_default) +The built-in [`anonymous` token](/consul/docs/security/acl/acl-tokens#built-in-tokens). + + +The following table describes the available DNS lookups and required policies when ACLs are enabled: + +| Lookup | Type | Description | ACLs Required | +| --- | --- | --- | --- | +| `*.node.consul` | Node | Allows Consul to resolve DNS requests for the target node. Example: `.node.consul` | `node:read` | +| `*.service.consul`
    `*.connect.consul`
    `*.ingress.consul`
    `*.virtual.consul` |Service: standard | Allows Consul to resolve DNS requests for target service instances running on ACL-authorized nodes. Example: `.service.consul` | `service:read`
    `node:read` | + +> **Tutorials**: For hands-on guidance on how to configure an appropriate token for DNS, refer to the tutorial for [Production Environments](/consul/tutorials/security/access-control-setup-production#token-for-dns) and [Development Environments](/consul/tutorials/day-0/access-control-setup#enable-acls-on-consul-clients). + +## Node lookups +Specify the name of the node, datacenter, and domain using the following FQDN syntax: + +```text +.node[..dc]. +``` + +The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of the agent. + +By default, the domain is `consul`. Refer to [Configure DNS Behaviors]() for information about using alternate domains. + +### Node lookup results + +Node lookups return A and AAAA records that contain the IP address and TXT records containing the `node_meta` values of the node. + +By default, TXT record values match the node's metadata key-value pairs according to [RFC1464](https://www.ietf.org/rfc/rfc1464.txt). If the metadata key starts with `rfc1035-`, the TXT record only includes the node's metadata value. + +The following example lookup queries the `foo` node in the `default` datacenter: + +```shell-session +$ dig @127.0.0.1 -p 8600 foo.node.consul ANY + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 foo.node.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 24355 +;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 0 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;foo.node.consul. IN ANY + +;; ANSWER SECTION: +foo.node.consul. 0 IN A 10.1.10.12 +foo.node.consul. 0 IN TXT "meta_key=meta_value" +foo.node.consul. 0 IN TXT "value only" + + +;; AUTHORITY SECTION: +consul. 0 IN SOA ns.consul. postmaster.consul. 1392836399 3600 600 86400 0 +``` + +### Node lookups for Consul Enterprise + +Consul Enterprise includes the admin partition concept, which is an abstraction that lets you define isolated administrative network areas. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information. + +Consul nodes reside in admin partitions within a datacenter. By default, node lookups query the same partition and datacenter of the Consul agent that received the DNS query. + +Use the following query format to specify a partition for a node lookup: + +``` +.node[..ap][..dc]. +``` + +Consul server agents are in the `default` partition. If you send a DNS query to Consul server agents, you must explicitly specify the partition of the target node if it is not `default`. + +## Service lookups +You can query the network for service providers using either the [standard lookup](#standard-lookup) method or [strict RFC 2782 lookup](#rfc-2782-lookup) method. + +By default, all SRV records are weighted equally in service lookup responses, but you can configure the weights using the [`Weights`](/consul/docs/services/configuration/services-configuration-reference#weights) attribute of the service definition. Refer to [Define Services](/consul/docs/services/usage/define-services) for additional information. + +The DNS protocol limits the size of requests, even when performing DNS TCP queries, which may affect your experience querying for services. For services with more than 500 instances, you may not be able to retrieve the complete list of instances for the service. Refer to [RFC 1035, Domain Names - Implementation and Specification](https://datatracker.ietf.org/doc/html/rfc1035#section-2.3.4) for additional information. + +Consul randomizes DNS SRV records and ignores weights specified in service configurations when printing responses. If records are truncated, each client using weighted SRV responses may have partial and inconsistent views of instance weights. As a result, the request distribution may be skewed from the intended weights. We recommend calling the [`/catalog/nodes` API endpoint](/consul/api-docs/catalog#list-nodes) to retrieve the complete list of nodes. You can apply query parameters to API calls to sort and filter the results. + +### Standard lookups +To perform standard service lookups, specify tags, the name of the service, datacenter, and domain using the following syntax to query for service providers: + +```text +[.].service[.].dc. +``` + +The `tag` subdomain is optional. It filters responses so that only service providers containing the tag appear. + +The `datacenter` subdomain is optional. By default, Consul interrogates the querying agent's datacenter. + +By default, the lookups query in the `consul` domain. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for information about using alternate domains. + +#### Standard lookup results +Standard services queries return A and SRV records. SRV records include the port that the service is registered on. SRV records are only served if the client specifically requests them. + +Services that fail their health check or that fail a node system check are omitted from the results. As a load balancing measure, Consul randomizes the set of nodes returned in the response. These mechanisms help you use DNS with application-level retries as the foundation for a self-healing service-oriented architecture. + +The following example retrieves the SRV records for any `redis` service registered in Consul. + +```shell-session +$ dig @127.0.0.1 -p 8600 consul.service.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 consul.service.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 +;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;consul.service.consul. IN SRV + +;; ANSWER SECTION: +consul.service.consul. 0 IN SRV 1 1 8300 foobar.node.dc1.consul. + +;; ADDITIONAL SECTION: +foobar.node.dc1.consul. 0 IN A 10.1.10.12 +``` + +The following example command and FQDN retrieves the SRV records for the primary Postgres service in the secondary datacenter: + +```shell-session hideClipboard +$ dig @127.0.0.1 -p 8600 primary.postgresql.service.dc2.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 primary.postgresql.service.dc2.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 +;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;consul.service.consul. IN SRV + +;; ANSWER SECTION: +consul.service.consul. 0 IN SRV 1 1 5432 primary.postgresql.service.dc2.consul. + +;; ADDITIONAL SECTION: +primary.postgresql.service.dc2.consul. 0 IN A 10.1.10.12 +``` + +### RFC 2782 lookup +Per [RFC 2782](https://tools.ietf.org/html/rfc2782), SRV queries must prepend `service` and `protocol` values with an underscore (`_`) to prevent DNS collisions. Use the following syntax to perform RFC 2782 lookups: + +```text +_._[.service][.]. +``` + +You can create lookups that filter results by placing service tags in the `protocol` field. Use the following syntax to create RFC 2782 lookups that filter results based on service tags: + +```text +_._[.service][.]. +``` + +The following example queries the `rabbitmq` service tagged with `amqp`, which returns an instance at `rabbitmq.node1.dc1.consul` on port `5672`: + +```shell-session +$ dig @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 52838 +;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;_rabbitmq._amqp.service.consul. IN SRV + +;; ANSWER SECTION: +_rabbitmq._amqp.service.consul. 0 IN SRV 1 1 5672 rabbitmq.node1.dc1.consul. + +;; ADDITIONAL SECTION: +rabbitmq.node1.dc1.consul. 0 IN A 10.1.11.20 +``` +#### SRV responses for hosts in the .addr subdomain + +If a service registered with Consul is configured with an explicit IP address or addresses in the [`address`](/consul/docs/services/configuration/services-configuration-reference#address) or [`tagged_address`](/consul/docs/services/configuration/services-configuration-reference#tagged_address) parameter, then Consul returns the hostname in the target field of the answer section for the DNS SRV query according to the following format: + +```text +.addr..consul`. +``` + +In the following example, the `rabbitmq` service is registered with an explicit IPv4 address of `192.0.2.10`. + +```hcl +node_name = "node1" + +services { + name = "rabbitmq" + address = "192.0.2.10" + port = 5672 +} +{ + "node_name": "node1", + "services": [ + { + "name": "rabbitmq", + "address": "192.0.2.10", + "port": 5672 + } + ] +} +``` + +The following example SRV query response contains a single record with a hostname written as a hexadecimal value: + +```shell-session +$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short +1 1 5672 c000020a.addr.dc1.consul. +``` + +You can convert hex octets to decimals to reveal the IP address. The following example command converts the hostname expressed as `c000020a` into the IPv4 address specified in the service registration. + +``` +$ echo -n "c000020a" | perl -ne 'printf("%vd\n", pack("H*", $_))' +192.0.2.10 +``` + +In the following example, the `rabbitmq` service is registered with an explicit IPv6 address of `2001:db8:1:2:cafe::1337`. + +```hcl +node_name = "node1" + +services { + name = "rabbitmq" + address = "2001:db8:1:2:cafe::1337" + port = 5672 +} +{ + "node_name": "node1", + "services": [ + { + "name": "rabbitmq", + "address": "2001:db8:1:2:cafe::1337", + "port": 5672 + } + ] +} +``` + +The following example SRV query response contains a single record with a hostname written as a hexadecimal value: + +```shell-session +$ dig @127.0.0.1 -p 8600 -t SRV _rabbitmq._tcp.service.consul +short +1 1 5672 20010db800010002cafe000000001337.addr.dc1.consul. +``` + +The response contains the fully-expanded IPv6 address with colon separators removed. The following command re-adds the colon separators to display the fully expanded IPv6 address that was specified in the service registration. + +```shell-session +$ echo -n "20010db800010002cafe000000001337" | perl -ne 'printf join(":", unpack("(A4)*", $_))."\n"' +2001:0db8:0001:0002:cafe:0000:0000:1337 +``` + +### Service lookups for Consul Enterprise +You can perform the following types of service lookups to query for services in another namespace, partition, and datacenter: + +- `.service` +- `.connect` +- `.virtual` +- `.ingress` + +Use the following query format to specify namespace, partition, or datacenter: +``` +[.].service[..ns][..ap][..dc] +``` + +The `namespace`, `partition`, and `datacenter` are optional. By default, all service lookups use the `default` namespace within the partition and datacenter of the Consul agent that received the DNS query. + +Consul server agents reside in the `default` partition. If DNS queries are addressed to Consul server agents, you must explicitly specify the partition of the target service when querying for services in partitions other than `default`. + +To lookup services imported from a cluster peer, refer to [Service virtual IP lookups for Consul Enterprise](#service-virtual-ip-lookups-for-consul-enterprise). + +#### Alternative formats for specifying namespace + +Although we recommend using the format described in [Service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise) for readability, you can use the alternate query format to specify namespaces but not partitions: + +``` +[.].service... +``` + +### Service mesh-enabled service lookups + +Add the `.connect` subdomain to query for service mesh-enabled services: + +```text +.connect. +``` + +This finds all service mesh-capable endpoints for the service. A service mesh-capable endpoint may be a proxy for a service or a natively integrated service mesh application. The DNS interface does not differentiate the two. + +Many services use a proxy that handles service discovery automatically. As a result, they may not use the DNS format, which is primarily for service mesh-native applications. +This endpoint only finds services within the same datacenter and does not support tags. Refer to the [`catalog` API endpoint](/consul/api-docs/catalog) for more complex behaviors. + +### Service virtual IP lookups + +Add the `.virtual` subdomain to queries to find the unique virtual IP allocated for a service: + +```text +.virtual[.]. +``` + +This returns the unique virtual IP for any service mesh-capable service. Each service mesh service has a virtual IP assigned to it by Consul. Sidecar proxies use the virtual IP to enable the [transparent proxy](/consul/docs/connect/transparent-proxy) feature. + +The peer name is an optional. The DNS uses it to query for the virtual IP of a service imported from the specified peer. + +Consul adds virtual IPs to the [`tagged_addresses`](/consul/services/configuration/services-configuration-reference#tagged-addresses) field in the service definition under the `consul-virtual` tag. + +#### Service virtual IP lookups for Consul Enterprise + +By default, a service virtual IP lookup checks the `default` namespace within the partition and datacenter of the Consul agent that received the DNS query. +To lookup services imported from a partition in another cluster peered to the querying cluster or open-source datacenter, specify the namespace and peer name in the lookup: + +```text +.virtual[.].. +``` + +To lookup services in a cluster peer that have not been imported, refer to [Service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise). + +### Ingress Service Lookups + +Add the `.ingress` subdomain to your DNS FQDN to find ingress-enabled services: + +```text +.ingress. +``` + +This finds all ingress gateway endpoints for the service. + +This endpoint finds services within the same datacenter and does not support tags. Refer to the [`catalog` API endpoint](/consul/api-docs/catalog) for more complex behaviors. + +### UDP-based DNS queries + +When the DNS query is performed using UDP, Consul truncateß the results without setting the truncate bit. This prevents a redundant lookup over TCP that generates additional load. If the lookup is done over TCP, the results are not truncated. \ No newline at end of file diff --git a/website/content/docs/services/services.mdx b/website/content/docs/services/services.mdx new file mode 100644 index 000000000000..3a2f21538907 --- /dev/null +++ b/website/content/docs/services/services.mdx @@ -0,0 +1,39 @@ +--- +layout: docs +page_title: Services Overview +description: >- + Learn about services and service discovery workflows and concepts for virtual machine environments. +--- + +# Services Overview +This topic provides overview information about services and how to make them discoverable in Consul when your network operates on virtual machines. If service mesh is enabled in your network, refer to the topics in [Consul Service Mesh](/consul/docs/connect/). If your network is running in Kubernetes, refer to the topics in [Consul on Kubernetes](/consul/docs/k8s). + +## Introduction +A _service_ is an entity in your network that performs a specialized operation or set of related operations. In many contexts, a service is software that you want to make available to users or other programs with access to your network. Services can also refer to native Consul functionality, such as _service mesh proxies_ and _gateways_, that enable you to establish connections between different parts of your network. + +You can define and register services with Consul, which makes them discoverable to other services in the network. You can also define various types of health checks that perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. + +## Workflow +For service discovery, the core Consul workflow for services consists of three stages: + +1. **Define services and health checks**: A service definition lets you define various aspects of the service, including how it is discovered by other services in the network. You can define health checks in the service definitions to verify the health of the service. Refer to [Define Services](/consul/docs/services/usage/define-services) and [Define Health Checks](/consul/docs/services/usage/checks) for additional information. + +1. **Register services and health checks**: After defining your services and health checks, you must register them with a Consul agent. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for additional information. + +1. **Query for services**: After registering your services and health checks, other services in your network can use the DNS to perform static or dynamic lookups to access your service. Refer to [DNS Usage Overview](/consul/docs/services/discovery/dns-overview) for additional information about the different ways to discover services in your datacenters. + + +## Service mesh use cases +Consul redirects service traffic through sidecar proxies if you use Consul service mesh. As a result, you must specify upstream configurations in service definitions. The service mesh experience is different for virtual machine (VM) and Kubernetes environments. + +### Virtual machines +You must define upstream services in the service definition. Consul uses the upstream configuration to bind the service with its upstreams. After registering the service, you must start a sidecar proxy on the VM to enable mesh connectivity. Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for details. + +### Kubernetes +If you use Consul on Kubernetes, enable the service mesh injector in your Consul Helm chart and Consul automatically adds a sidecar to each of your pods using the Kubernetes `Service` definition as a reference. You can specify upstream annotations in the `Deployment` definition to bind upstream services to the pods. +Refer to [`connectInject`](/consul/docs/k8s/connect#installation-and-configuration) and [the upstreams annotation documentation](/consul/docs/k8s/annotations-and-labels#consul-hashicorp-com-connect-service-upstreams) for additional information. + +### Multiple services +You can define common characteristics for services in your mesh, such as the admin partition, namespace, or upstreams, by creating and applying a `service-defaults` configuration entry. You can also define override configurations for specific upstreams or service instances. To use `service-defaults` configuraiton entries, you must enable Consul service mesh in your network. + +Refer to [Define Service Defaults](/consul/docs/services/usage/define-services#define-service-defaults) for additional information. \ No newline at end of file diff --git a/website/content/docs/services/usage/checks.mdx b/website/content/docs/services/usage/checks.mdx new file mode 100644 index 000000000000..99d26c722292 --- /dev/null +++ b/website/content/docs/services/usage/checks.mdx @@ -0,0 +1,592 @@ +--- +layout: docs +page_title: Define Health Checks +description: -> + Learn how to configure different types of health checks for services you register with Consul. +--- + +# Define Health Checks +This topic describes how to create different types of health checks for your services. + + +## Overview +Health checks are configurations that verifies the health of a service or node. Health checks configurations are nested in the `service` block. Refer to [Define Services](/consul/docs/services/usage/define-services) for information about specifying other service parameters. + +You can define individual health checks for your service in separate `check` blocks or define multiple checks in a `checks` block. Refer to [Define multiple checks](#define-multiple-checks) for additional information. + +You can create several different kinds of checks: + +- _Script_ checks invoke an external application that performs the health check, exits with an appropriate exit code, and potentially generates output. Script checks are one of the most common types of checks. +- _HTTP_ checks make an HTTP GET request to the specified URL and wait for the specified amount of time. HTTP checks are one of the most common types of checks. +- _TCP_ checks attempt to connect to an IP or hostname and port over TCP and wait for the specified amount of time. +- _UDP_ checks send UDP datagrams to the specified IP or hostname and port and wait for the specified amount of time. +- _Time-to-live (TTL)_ checks are passive checks that await updates from the service. If the check does not receive a status update before the specified duration, the health check enters a `critical`state. +- _Docker_ checks are dependent on external applications packaged with a Docker container that are triggered by calls to the Docker `exec` API endpoint. +- _gRPC_ checks probe applications that support the standard gRPC health checking protocol. +- _H2ping_ checks test an endpoint that uses http2. The check connects to the endpoint and sends a ping frame. +- _Alias_ checks represent the health state of another registered node or service. + +If your network runs in a Kubernetes environment, you can sync service health information with Kubernetes health checks. Refer to [Configure Health Checks for Consul on Kubernetes](/consul/docs/k8s/connect/health) for details. + +### Registration + +After defining health checks, you must register the service containing the checks with Consul. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for additional information. If the service is already registered, you can reload the service configuration file to implement your health check. Refer to [Reload](/consul/commands/reload) for additional information. + +## Define multiple checks + +You can define multiple checks for a service in a single `checks` block. The `checks` block contains an array of objects. The objects contain the configuration for each health check you want to implement. The following example includes two script checks named `mem` and `cpu` and an HTTP check that calls the `/health` API endpoint. + + + +```hcl +checks = [ + { + id = "chk1" + name = "mem" + args = ["/bin/check_mem", "-limit", "256MB"] + interval = "5s" + }, + { + id = "chk2" + name = "/health" + http = "http://localhost:5000/health" + interval = "15s" + }, + { + id = "chk3" + name = "cpu" + args = ["/bin/check_cpu"] + interval = "10s" + }, + ... +] +``` + +```json +{ + "checks": [ + { + "id": "chk1", + "name": "mem", + "args": ["/bin/check_mem", "-limit", "256MB"], + "interval": "5s" + }, + { + "id": "chk2", + "name": "/health", + "http": "http://localhost:5000/health", + "interval": "15s" + }, + { + "id": "chk3", + "name": "cpu", + "args": ["/bin/check_cpu"], + "interval": "10s" + }, + ... + ] +} +``` + + + +## Define initial health check status +When checks are registered against a Consul agent, they are assigned a `critical` status by default. This prevents services from registering as `passing` and entering the service pool before their health is verified. You can add the `status` parameter to the check definition to specify the initial state. In the following example, the check registers in a `passing` state: + + + +```hcl +check = { + id = "mem" + args = ["/bin/check_mem", "-limit", "256MB"] + interval = "10s" + status = "passing" +} +``` + +```json +{ + "check": [ + { + "args": [ + "/bin/check_mem", + "-limit", + "256MB" + ], + "id": "mem", + "interval": "10s", + "status": "passing" + } + ] +} +``` + + + +## Script checks +Script checks invoke an external application that performs the health check, exits with an appropriate exit code, and potentially generates output data. The output of a script check is limited to 4KB. Outputs that exceed the limit are truncated. + +Script checks timeout after 30 seconds by default, but you can configure a custom script check timeout value by specifying the `timeout` field in the check definition. When the timeout is reached on Windows, Consul waits for any child processes spawned by the script to finish. For any other system, Consul attempts to force-kill the script and any child processes it has spawned once the timeout has passed. + +### Script check configuration +To enable script checks, you must first enable the agent to send external requests, then configure the health check settings in the service definition: + +1. Add one of the following configurations to your agent configuration file to enable a script check: + - [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#enable_local_script_checks): Enable script checks defined in local configuration files. Script checks registered using the HTTP API are not allowed. + - [`enable_script_checks`](/consul/docs/agent/config/cli-flags#enable_script_checks): Enable script checks no matter how they are registered. + + !> **Security warning:** Enabling non-local script checks in some configurations may introduce a known remote execution vulnerability targeted by malware. We strongly recommend `enable_local_script_checks` instead. + +1. Specify the script to run in the `args` of the `check` block in your service configuration file. In the following example, a check named `Memory utilization` invokes the `check_mem.py` script every 10 seconds and times out if a response takes longer than one second: + + + + ```hcl + service { + ## ... + check = { + id = "mem-util" + name = "Memory utilization" + args = ["/usr/local/bin/check_mem.py", "-limit", "256MB"] + interval = "10s" + timeout = "1s" + } + } + ``` + + ```json + { + "service": [ + { + "check": { + "id": "mem-util", + "name": "Memory utilization", + "args": ["/usr/local/bin/check_mem.py", "-limit", "256MB"], + "interval": "10s", + "timeout": "1s" + } + } ] + } + ``` + + +Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +### Script check exit codes +The following exit codes returned by the script check determine the health check status: + +- Exit code 0 - Check is passing +- Exit code 1 - Check is warning +- Any other code - Check is failing + +Any output of the script is captured and made available in the `Output` field of checks included in HTTP API responses. Refer to the example described in the [local service health endpoint](/consul/api-docs/agent/service#by-name-json). + +## HTTP checks +_HTTP_ checks send an HTTP request to the specified URL and report the service health based on the [HTTP response code](#http-check-response-codes). We recommend using HTTP checks over [script checks](#script-checks) that use cURL or another external process to check an HTTP operation. + +### HTTP check configuration +Add an `http` field to the `check` block in your service definition file and specify the HTTP address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an HTTP check named `HTTP API on port 5000` sends a `POST` request to the `health` endpoint every 10 seconds: + + + +```hcl +check = { + id = "api" + name = "HTTP API on port 5000" + http = "https://localhost:5000/health" + tls_server_name = "" + tls_skip_verify = false + method = "POST" + header = { + Content-Type = ["application/json"] + } + body = "{\"method\":\"health\"}" + disable_redirects = true + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "api", + "name": "HTTP API on port 5000", + "http": "https://localhost:5000/health", + "tls_server_name": "", + "tls_skip_verify": false, + "method": "POST", + "header": { "Content-Type": ["application/json"] }, + "body": "{\"method\":\"health\"}", + "interval": "10s", + "timeout": "1s" + } +} +``` + + +HTTP checks send GET requests by default, but you can specify another request method in the `method` field. You can send additional headers in the `header` block. The `header` block contains a key and an array of strings, such as `{"x-foo": ["bar", "baz"]}`. By default, HTTP checks timeout at 10 seconds, but you can specify a custom timeout value in the `timeout` field. + +HTTP checks expect a valid TLS certificate by default. You can disable certificate verification by setting the `tls_skip_verify` field to `true`. When using TLS and a host name is specified in the `http` field, the check automatically determines the SNI from the URL. If the `http` field is configured with an IP address or if you want to explicitly set the SNI, specify the name in the `tls_server_name` field. + +The check follows HTTP redirects configured in the network by default. Set the `disable_redirects` field to `true` to disable redirects. + +### HTTP check response codes +Responses larger than 4KB are truncated. The HTTP response determines the status of the service: + +- A `200`-`299` response code is healthy. +- A `429` response code indicating too many requests is a warning. +- All other response codes indicate a failure. + + +## TCP checks +TCP checks establish connections to the specified IPs or hosts. If the check successfully establishes a connection, the service status is reported as `success`. If the IP or host does not accept the connection, the service status is reported as `critical`. We recommend TCP checks over [script checks](#script-checks) that use netcat or another external process to check a socket operation. + +### TCP check configuration +Add a `tcp` field to the `check` block in your service definition file and specify the address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/health-checks-configuration) for information about all health check configurations. + +In the following example, a TCP check named `SSH TCP on port 22` attempts to connect to `localhost:22` every 10 seconds: + + + + +```hcl +check = { + id = "ssh" + name = "SSH TCP on port 22" + tcp = "localhost:22" + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "ssh", + "name": "SSH TCP on port 22", + "tcp": "localhost:22", + "interval": "10s", + "timeout": "1s" + } +} +``` + + + +If a hostname resolves to an IPv4 and an IPv6 address, Consul attempts to connect to both addresses. The first successful connection attempt results in a successful check. + +By default, TCP check requests timeout at 10 seconds, but you can specify a custom timeout in the `timeout` field. + +## UDP checks +UDP checks direct the Consul agent to send UDP datagrams to the specified IP or hostname and port. The check status is set to `success` if any response is received from the targeted UDP server. Any other result sets the status to `critical`. + +### UDP check configuration +Add a `udp` field to the `check` block in your service definition file and specify the address, including port number, for sending datagrams. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, a UDP check named `DNS UDP on port 53` sends datagrams to `localhost:53` every 10 seconds: + + + +```hcl +check = { + id = "dns" + name = "DNS UDP on port 53" + udp = "localhost:53" + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "dns", + "name": "DNS UDP on port 53", + "udp": "localhost:53", + "interval": "10s", + "timeout": "1s" + } +} +``` + + + +By default, UDP checks timeout at 10 seconds, but you can specify a custom timeout in the `timeout` field. If any timeout on read exists, the check is still considered healthy. + +## OSService check +OSService checks if an OS service is running on the host. OSService checks support Windows services on Windows hosts or SystemD services on Unix hosts. The check logs the service as `healthy` if it is running. If the service is not running, the status is logged as `critical`. All other results are logged with `warning`. A `warning` status indicates that the check is not reliable because an issue is preventing it from determining the health of the service. + +### OSService check configurations +Add an `os_service` field to the `check` block in your service definition file and specify the name of the service to check. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference] for information about all health check configurations. + +In the following example, an OSService check named `svcname-001 Windows Service Health` verifies that the `myco-svctype-svcname-001` service is running every 10 seconds: + + + +```hcl +check = { + id = "myco-svctype-svcname-001" + name = "svcname-001 Windows Service Health" + service_id = "flash_pnl_1" + os_service = "myco-svctype-svcname-001" + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "myco-svctype-svcname-001", + "name": "svcname-001 Windows Service Health", + "service_id": "flash_pnl_1", + "os_service": "myco-svctype-svcname-001", + "interval": "10s" + } +} +``` + + + +## TTL checks +Time-to-live (TTL) checks wait for an external process to report the service's state to a Consul [`/agent/check` HTTP endpoint](/consul/api-docs/agent/check). If the check does not receive an update before the specified `ttl` duration, the check logs the service as `critical`. For example, if a healthy application is configured to periodically send a `PUT` request a status update to the HTTP endpoint, then the health check logs a `critical` state if the application is unable to send the update before the TTL expires. The check uses the following endpoints to update health information: + +- [pass](/consul/api-docs/agent/check#ttl-check-pass) +- [warn] (/consul/api-docs/agent/check#ttl-check-warn) +- [fail](/consul/api-docs/agent/check#ttl-check-fail) +- [update](/consul/api-docs/agent/check#ttl-check-update) + +TTL checks also persist their last known status to disk so that the Consul agent can restore the last known status of the check across restarts. Persisted check status is valid through the end of the TTL from the time of the last check. + +You can manually mark a service as unhealthy using the [`consul maint` CLI command](/consul/commands/maint) or [`agent/maintenance` HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode), rather than waiting for a TTL health check if the `ttl` duration is high. + +### TTL check configuration +Add a `ttl` field to the `check` block in your service definition file and specify how long to wait for an update from the external process. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference] for information about all health check configurations. + +In the following example, a TTL check named `Web App Status` logs the application as `critical` if a status update is not received every 30 seconds: + + + + +```hcl +check = { + id = "web-app" + name = "Web App Status" + notes = "Web app does a curl internally every 10 seconds" + ttl = "30s" +} +``` + +```json +{ + "check": { + "id": "web-app", + "name": "Web App Status", + "notes": "Web app does a curl internally every 10 seconds", + "ttl": "30s" + } +} +``` + + + +## Docker checks +Docker checks invoke an application packaged within a Docker container. The application should perform a health check and exit with an appropriate exit code. + +The application is triggered within the running container through the Docker `exec` API. You should have access to either the Docker HTTP API or the Unix socket. Consul uses the `$DOCKER_HOST` environment variable to determine the Docker API endpoint. + +The output of a Docker check is limited to 4KB. Larger outputs are truncated. + +### Docker check configuration +To enable Docker checks, you must first enable the agent to send external requests, then configure the health check settings in the service definition: + +1. Add one of the following configurations to your agent configuration file to enable a Docker check: + + - [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#enable_local_script_checks): Enable script checks defined in local config files. Script checks registered using the HTTP API are not allowed. + + - [`enable_script_checks`](/consul/docs/agent/config/cli-flags#enable_script_checks): Enable script checks no matter how they are registered. + + !> **Security warning**: Enabling non-local script checks in some configurations may introduce a known remote execution vulnerability targeted by malware. We strongly recommend `enable_local_script_checks` instead. +1. Configure the following fields in the `check` block in your service definition file: + - `docker_container_id`: The `docker ps` command is a common way to get the ID. + - `shell`: Specifies the shell to use for performing the check. Different containers can run different shells on the same host. + - `args`: Specifies the external application to invoke. + - `interval`: Specifies the interval for running the check. + +In the following example, a Docker check named `Memory utilization` invokes the `check_mem.py` application in container `f972c95ebf0e` every 10 seconds: + + + + +```hcl +check = { + id = "mem-util" + name = "Memory utilization" + docker_container_id = "f972c95ebf0e" + shell = "/bin/bash" + args = ["/usr/local/bin/check_mem.py"] + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Memory utilization", + "docker_container_id": "f972c95ebf0e", + "shell": "/bin/bash", + "args": ["/usr/local/bin/check_mem.py"], + "interval": "10s" + } +} +``` + + + +## gRPC checks +gRPC checks send a request to the specified endpoint. These checks are intended for applications that support the standard [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + +### gRPC check configuration +Add a `grpc` field to the `check` block in your service definition file and specify the endpoint, including port number, for sending requests. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference] for information about all health check configurations. + +In the following example, a gRPC check named `Service health status` probes the entire application by sending requests to `127.0.0.1:12345` every 10 seconds: + + + +```hcl +check = { + id = "mem-util" + name = "Service health status" + grpc = "127.0.0.1:12345" + grpc_use_tls = true + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Service health status", + "grpc": "127.0.0.1:12345", + "grpc_use_tls": true, + "interval": "10s" + } +} +``` + + + +gRPC checks probe the entire gRPC server, but you can check on a specific service by adding the service identifier after the gRPC check's endpoint using the following format: `/:service_identifier`. + +In the following example, a gRPC check probes `my_service` in the application at `127.0.0.1:12345` every 10 seconds: + + + + +```hcl +check = { + id = "mem-util" + name = "Service health status" + grpc = "127.0.0.1:12345/my_service" + grpc_use_tls = true + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Service health status", + "grpc": "127.0.0.1:12345/my_service", + "grpc_use_tls": true, + "interval": "10s" + } +} +``` + + + +TLS is disabled for gRPC checks by default. You can enable TLS by setting `grpc_use_tls` to `true`. If TLS is enabled, you must either provide a valid TLS certificate or disable certificate verification by setting the `tls_skip_verify` field to `true`. + +By default, gRPC checks timeout after 10 seconds, but you can specify a custom duration in the `timeout` field. + +## H2ping checks +H2ping checks test an endpoint that uses HTTP2 by connecting to the endpoint and sending a ping frame. If the endpoint sends a response within the specified interval, the check status is set to `success`. + +### H2ping check configuration +Add an `h2ping` field to the `check` block in your service definition file and specify the HTTP2 endpoint, including port number, for the check to ping. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an H2ping check named `h2ping` pings the endpoint at `localhost:22222` every 10 seconds: + + + + +```hcl +check = { + id = "h2ping-check" + name = "h2ping" + h2ping = "localhost:22222" + interval = "10s" + h2ping_use_tls = false +} +``` + +```json +{ + "check": { + "id": "h2ping-check", + "name": "h2ping", + "h2ping": "localhost:22222", + "interval": "10s", + "h2ping_use_tls": false + } +} +``` + + + +TLS is enabled by default, but you can disable TLS by setting `h2ping_use_tls` to `false`. When TLS is disabled, the Consul sends pings over h2c. When TLS is enabled, a valid certificate is required unless `tls_skip_verify` is set to `true`. + +By default, H2ping checks timeout at 10 seconds, but you can specify a custom duration in the `timeout` field. + + +## Alias checks +Alias checks continuously report the health state of another registered node or service. If the alias experiences errors while watching the actual node or service, the check reports a`critical` state. Consul updates the alias and actual node or service state asynchronously but nearly instantaneously. + +For aliased services on the same agent, the check monitors the local state without consuming additional network resources. For services and nodes on different agents, the check maintains a blocking query over the agent's connection with a current server and allows stale requests. + +### ACLs +For the blocking query, the alias check presents the ACL token set on the actual service or the token configured in the check definition. If neither are available, the alias check falls back to the default ACL token set for the agent. Refer to [`acl.tokens.default`](/consul/docs/agent/config/config-files#acl_tokens_default) for additional information about the default ACL token. + +### Alias checks configuration +Add an `alias_service` field to the `check` block in your service definition file and specify the name of the service or node to alias. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference] for information about all health check configurations. + +In the following example, an alias check with the ID `web-alias` reports the health state of the `web` service: + + + + +```hcl +check = { + id = "web-alias" + alias_service = "web" +} +``` + +```json +{ + "check": { + "id": "web-alias", + "alias_service": "web" + } +} +``` + + + +By default, the alias must be registered with the same Consul agent as the alias check. If the service is not registered with the same agent, you must specify `"alias_node": ""` in the `check` configuration. If no service is specified and the `alias_node` field is enabled, the check aliases the health of the node. If a service is specified, the check will alias the specified service on this particular node. \ No newline at end of file diff --git a/website/content/docs/services/usage/define-services.mdx b/website/content/docs/services/usage/define-services.mdx new file mode 100644 index 000000000000..35df236668d2 --- /dev/null +++ b/website/content/docs/services/usage/define-services.mdx @@ -0,0 +1,366 @@ +--- +layout: docs +page_title: Define Services +description: >- + Learn how to define services so that they are discoverable in your network. +--- + +# Define Services + +This topic describes how to define services so that they can be discovered by other services. Refer to [Services Overview](/consul/docs/services/services) for additional information. + +## Overview + +You must tell Consul about the services deployed to your network if you want them to be discoverable. You can define services in a configuration file or send the service definition parameters as a payload to the `/agent/service/register` API endpoint. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details about how to register services with Consul. + +You can define multiple services individually using `service` blocks or group multiple services into the same `services` configuration block. Refer to [Define multiple services in a single file](#define-multiple-services-in-a-single-file) for additional information. + +If Consul service mesh is enabled in your network, you can use the `service-defaults` configuration entry to specify default global values for services. The configuraiton entry lets you define common service parameter, such as upstreams, namespaces, and partitions. Refer to [Define service defaults](#define-service-defaults) for additional information. + +## Requirements + +The core service discovery features are available in all versions of Consul. + +### Service defaults +To use the [service defaults configuration entry](#define-service-defaults), verify that your installation meets the following requirements: + +- Consul 1.5.0+ +- Consul 1.8.4+ is required to use the `ServiceDefaults` custom resource on Kubernetes + +### ACLs +If ACLs are enabled, resources in your network must present a token with `service:read` access to read a service defaults configuration entry. + +You must also present a token with `service:write` access to create, update, or delete a service defaults configuration entry. + +Service configurations must also contain and present an ACL token to perform anti-entropy syncs and deregistration operations. Refer to [Modify anti-entropy synchronozation](#modify-anti-entropy-synchronization) for additional information. + +On Consul Enterprise, you can register services with specific namespaces if the services' ACL tokens are scoped to the namespace. Services registered with a service definition do not inherit the namespace associated with the ACL token specified in the `token` field. The `namespace` and the `token` parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + +## Define a service +Create a file for your service configurations and add a `service` block. The `service` block contains the parameters that configure various aspects of the service, including how it is discovered by other services in the network. The only required parameter is `name`. Refer to [Service Definition Reference](/consul/docs/services/configuration/services-configuration-reference) for details about the configuration options. + +For Kubernetes environments, you can enable the [`connectInject`](/consul/docs/k8s/connect#installation-and-configuration) configuration in your Consul Helm chart so that Consul automatically adds a sidecar to each of your pods. Consul uses the Kubernetes `Service` definition as a reference. + +The following example defines a service named `redis` that is available on port `80`. By default, the service has the IP address of the agent node. + + + + +```hcl +service { + name = "redis" + id = "redis" + port = 80 + tags = ["primary"] + + meta = { + custom_meta_key = "custom_meta_value" + } + + tagged_addresses = { + lan = { + address = "192.168.0.55" + port = 8000 + } + + wan = { + address = "198.18.0.23" + port = 80 + } + } +} +``` + + + + +```json +{ + "service": [ + { + "id": "redis", + "meta": [ + { + "custom_meta_key": "custom_meta_value" + } + ], + "name": "redis", + "port": 80, + "tagged_addresses": [ + { + "lan": [ + { + "address": "192.168.0.55", + "port": 8000 + } + ], + "wan": [ + { + "address": "198.18.0.23", + "port": 80 + } + ] + } + ], + "tags": [ + "primary" + ] + } + ] +} +``` + + + +```yaml +service: +- id: redis + meta: + - custom_meta_key: custom_meta_value + name: redis + port: 80 + tagged_addresses: + - lan: + - address: 192.168.0.55 + port: 8000 + wan: + - address: 198.18.0.23 + port: 80 + tags: + - primary +``` + + + +### Health checks + +You can add a `check` or `checks` block to your service configuration to define one or more health checks that monitor the health of your services. Refer to [Define Health Checks](/consul/docs/services/usage/checks) for additional information. + +### Register a service + +You can register your service using the [`consul services` command](/consul/commands/services) or by calling the [`/agent/services` API endpoint](/consul/api-docs/agent/services). Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details. + +## Define service defaults +If Consul service mesh is enabled in your network, you can define default values for services in your mesh by creating and applying a `service-defaults` configuration entry containing. Refer to [Service Mesh Configuration Overview](/consul/docs/connect/configuration) for additional information. + +Create a file for the configuration entry and specify the required fields. If you are authoring `service-defaults` in HCL or JSON, the `Kind` and `Name` fields are required. On Kubernetes, the `apiVersion`, `kind`, and `metadata.name` fields are required. Refer to [Service Defaults Reference](/consul/docs/connect/config-entries/service-defaults) for details about the configuration options. + +The following example instructs services named `counting` to send up to `512` concurrent requests to a mesh gateway: + + + +```hcl +Kind = "service-defaults" +Name = "counting" + +UpstreamConfig = { + Defaults = { + MeshGateway = { + Mode = "local" + } + Limits = { + MaxConnections = 512 + MaxPendingRequests = 512 + MaxConcurrentRequests = 512 + } + } + + Overrides = [ + { + Name = "dashboard" + MeshGateway = { + Mode = "remote" + } + } + ] +} +``` +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: counting +spec: + upstreamConfig: + defaults: + meshGateway: + mode: local + limits: + maxConnections: 512 + maxPendingRequests: 512 + maxConcurrentRequests: 512 + overrides: + - name: dashboard + meshGateway: + mode: remote +``` +```json +{ + "Kind": "service-defaults", + "Name": "counting", + "UpstreamConfig": { + "Defaults": { + "MeshGateway": { + "Mode": "local" + }, + "Limits": { + "MaxConnections": 512, + "MaxPendingRequests": 512, + "MaxConcurrentRequests": 512 + } + }, + "Overrides": [ + { + "Name": "dashboard", + "MeshGateway": { + "Mode": "remote" + } + } + ] + } +} +``` + + +### Apply service defaults + +You can apply your `service-defaults` configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). In Kubernetes environments, apply the `service-defaults` custom resource definitions (CRD) to implement and manage Consul configuration entries. + +Refer to the following topics for details about applying configuration entries: +- [How to Use Configuration Entries](/consul/docs/agent/config-entries) +- [Custom Resource Definitions for Consul on Kubernetes](/consul/docs/k8s/crds) + +## Define multiple services in a single file + +The `services` block contains an array of `service` objects. It is a wrapper that enables you to define multiple services in the service definition and instruct Consul to expect more than just a single service configuration. As a result, you can register multiple services in a single `consul services register` command. Note that the `/agent/service/register` API endpoint does not support the `services` parameter. + +In the following example, the service definition configures an instance of the `redis` service tagged as `primary` running on port `6000`. It also configures an instance of the service tagged as `secondary` running on port `7000`. + + + + + +```hcl +services { + id = "red0" + name = "redis" + tags = [ + "primary" + ] + address = "" + port = 6000 + checks = [ + { + args = ["/bin/check_redis", "-p", "6000"] + interval = "5s" + timeout = "20s" + } + ] +} +services { + id = "red1" + name = "redis" + tags = [ + "delayed", + "secondary" + ] + address = "" + port = 7000 + checks = [ + { + args = ["/bin/check_redis", "-p", "7000"] + interval = "30s" + timeout = "60s" + } + ] +} + +``` + + + + + +```json +{ + "services": [ + { + "id": "red0", + "name": "redis", + "tags": [ + "primary" + ], + "address": "", + "port": 6000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "6000"], + "interval": "5s", + "timeout": "20s" + } + ] + }, + { + "id": "red1", + "name": "redis", + "tags": [ + "delayed", + "secondary" + ], + "address": "", + "port": 7000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "7000"], + "interval": "30s", + "timeout": "60s" + } + ] + }, + ... + ] +} +``` + + + + +## Modify anti-entropy synchronization + +By default, the Consul agent uses anti-entropy mechanisms to maintain information about services and service health, and synchronize local states with the Consul catalog. You can enable the `enable_tag_override` option in the service configuration, which lets external agents change the tags for a service. This can be useful in situations where an external monitoring service needs to be the source of truth for tag information. Refer [Anti-entropy](/consul/docs/architecture/anti-entropy) for details. + +Add the `enable_tag_override` option to the `service` block and set the value to `true`: + + + + +```hcl +service { + ## ... + enable_tag_override = true + ## ... +} +``` + +```json +"service": { + ## ... + "enable_tag_override": true, + ## ... +} +``` + + + +This configuration only applies to the locally registered service. Nodes that register the same service apply the `enable_tag_override` and other service configurations independently. The tags for a service registered on one node update are not affected by operations performed on services with the same name registered on other nodes. + +Refer to [`enable_tag_override`](/consul/docs/services/configuration/services-configuration-reference#enable_tag_override) for additional configuration information. + +## Services in service mesh environments +Defining services for service mesh environments on virtual machines and in Kubernetes requires a different workflow. + +### Define service mesh proxies +You can register services to function as service mesh or sidecar proxies so that they can facilitate communication between other services across your network. Refer to [Service Mesh Proxy Overview](/consul/docs/connect/registration) for additional information. + +### Define services in Kubernetes +You can enable the services running in Kubernetes and Consul to sync automatically. Doing so ensures that Kubernetes services are available to Consul agents and services in Consul can be available as first-class Kubernetes services. Refer to [Service Sync for Consul on Kubernetes](/consul/docs/k8s/service-sync) for details. \ No newline at end of file diff --git a/website/content/docs/services/usage/register-services-checks.mdx b/website/content/docs/services/usage/register-services-checks.mdx new file mode 100644 index 000000000000..fca829a00e1c --- /dev/null +++ b/website/content/docs/services/usage/register-services-checks.mdx @@ -0,0 +1,67 @@ +--- +layout: docs +page_title: Register Services and Health Checks +description: -> + Learn how to register services and health checks with Consul agents. +--- + +# Register Services and Health Checks +This topic describes how to register services and health checks with Consul in networks running on virtual machines (VM). Refer to [Define Services](/consul/usage/services/usage/define-services) and [Define Health Checks](/consul/usage/services/usage/checks) for information about how to define services and health checks. + +## Overview +Register services and health checks in VM environments by providing the service definition to a Consul agent. You can use several different methods to register services and health checks. + +- Start a Consul agent and pass the service definition in the [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). +- Reload the a running Consul agent and pass the service definition in the [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). Use this method when implementing changes to an existing service or health check definition. +- Use the [`consul services register` command](/consul/commands/services/register) to register new service and health checks with a running Consul agent. +- Call the [`/agent/service/register`](/consul/api-docs/agent/service#register-service) HTTP API endpoint to to register new service and health checks with a running Consul agent. +- Call the [`/agent/check/register`](/consul/api-docs/agent/check#register-check) HTTP API endpoint to register a health check independent from the service. + +When a service is registered using the HTTP API endpoint or CLI command, the checks persist in the Consul data folder. If the agent restarts, Consul uses the service and check configurations in the configuration directory to start the services. + +Note that health checks associated with a service are application-level checks. + +## Start an agent +We recommend registering services on startup because the service persists if the agent fails. Specify the directory containing the service definition with the `-config-dir` option on startup. When the Consul agent starts, it processes all configurations in the directory and registers any services contained in the configurations. In the following example, the Consul agent starts and loads the configurations contained in the `configs` directory: + +```shell-session +$ consul agent -config-dir configs +``` + +Refer to [Starting the Consul Agent](/consul/docs/agent#starting-the-consul-agent) for additional information. + +## Reload an agent +Store your service definition file in the directory containing your Consul configuration files and either send a `SIGHUP` signal to the Consul agent service or run the `consul reload` command. Refer to [Consul Reload](/consul/commands/reload) for additional information. + +## Register using the CLI +Run the `consul services register` command and specify the service definition file to register the services and health checks defined in the file. In the following example, a service named `web` is registered: + +```shell-session +$ consul services register -name=web services.hcl +``` + +Refer to [Consul Agent Service Registration](/consul/commands/services/register) for additional information about using the command. + +## Register using the API + +Use the following methods to register services and health checks using the HTTP API. + +### Register services +Send a `PUT` request to the `/agent/service/register` API endpoint to dynamically register a service and its associated health checks. To register health checks independently, [call the checks API endpoint](#call-the-checks-http-api-endpoint). + +The following example request registers the service defined in the `service.json` file. + +```shell-session +$ curl --request PUT --data @service.json http://localhost:8500/v1/agent/service/register +``` + +Refer to [Service - Agent HTTP API](/consul/api-docs/agent/service) for additional information about the `services` endpoint. + +### Register health checks +Send a `PUT` request to the `/agent/check/register` API endpoint to dynamically register a health check to the local Consul agent. The following example request registers a health check defined in a `payload.json` file. + +```shell-session +$ curl --request PUT --data @payload.json http://localhost:8500/v1/agent/check/register +``` + +Refer to [Check - Agent HTTP API](/consul/api-docs/check/service) for additional information about the `check` endpoint. diff --git a/website/content/docs/troubleshoot/troubleshoot-services.mdx b/website/content/docs/troubleshoot/troubleshoot-services.mdx index 3451d2e50672..92a66881475a 100644 --- a/website/content/docs/troubleshoot/troubleshoot-services.mdx +++ b/website/content/docs/troubleshoot/troubleshoot-services.mdx @@ -13,7 +13,7 @@ For more information, refer to the [`consul troubleshoot` CLI documentation](/co ## Introduction -When communication between upstream and downstream services in a service mesh fails, you can diagnose the cause manually with one or more of Consul’s built-in features, including [health check queries](/consul/docs/discovery/checks), [the UI topology view](/consul/docs/connect/observability/ui-visualization), and [agent telemetry metrics](/consul/docs/agent/telemetry#metrics-reference). +When communication between upstream and downstream services in a service mesh fails, you can diagnose the cause manually with one or more of Consul’s built-in features, including [health check queries](/consul/docs/services/usage/checks), [the UI topology view](/consul/docs/connect/observability/ui-visualization), and [agent telemetry metrics](/consul/docs/agent/telemetry#metrics-reference). The `consul troubleshoot` command performs several checks in sequence that enable you to discover issues that impede service-to-service communication. The process systematically queries the [Envoy administration interface API](https://www.envoyproxy.io/docs/envoy/latest/operations/admin) and the Consul API to determine the cause of the communication failure. @@ -100,7 +100,7 @@ In the example output, troubleshooting upstream communication reveals that the ` ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found ``` -The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/discovery/dns#standard-lookup). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. +The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/services/discovery/dns-static-lookups#standard-lookup). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. For more information, refer to the [`consul troubleshoot` CLI documentation](/consul/commands/troubleshoot). diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 07d456ad816c..06997760e35c 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -990,11 +990,9 @@ config files loaded by Consul, even when using the [`-config-file`](/consul/docs/agent/config/cli-flags#_config_file) argument to specify a file directly. -#### Service Definition Parameter Case changed +#### Use Snake Case for Service Definition Parameters -All config file formats now require snake_case fields, so all CamelCased parameter -names should be changed before upgrading. -See [Service Definition Parameter Case](/consul/docs/discovery/services#service-definition-parameter-case) documentation for details. +Snake case, which is a convention that uses underscores between words in a configuration key, is required for all configuration file formats. Change any camel cased parameter to snake case equivalents before upgrading. #### Deprecated Options Have Been Removed diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 23cc2d5d1aa0..9b53bd05117d 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -303,19 +303,70 @@ "divider": true }, { - "title": "Service Discovery", + "title": "Services", "routes": [ { - "title": "Register Services - Service Definitions", - "path": "discovery/services" + "title": "Services Overview", + "path": "services/services" }, { - "title": "Find Services - DNS Interface", - "path": "discovery/dns" + "title": "Define and Register", + "routes": [ + { + "title": "Define Services", + "path": "services/usage/define-services" + }, + { + "title": "Define Health Checks", + "path": "services/usage/checks" + }, + { + "title": "Register Services and Health Checks", + "path": "services/usage/register-services-checks" + } + ] + }, + { + "title": "Service Discovery", + "routes": [ + { + "title": "DNS Usage Overview", + "path": "services/discovery/dns-overview" + }, + { + "title": "Configure Consul DNS Behavior", + "path": "services/discovery/dns-configuration" + }, + { + "title": "Perform Static DNS Lookups", + "path": "services/discovery/dns-static-lookups" + }, + { + "title": "Enable Dynamic DNS Lookups", + "path": "services/discovery/dns-dynamic-lookups" + } + ] }, { - "title": "Monitor Services - Check Definitions", - "path": "discovery/checks" + "title": "Configuration", + "routes": [ + { + "title": "Services Configuration Overview", + "path": "services/configuration/services-configuration-overview" + }, + { + "title": "Services Configuration Reference", + "path": "services/configuration/services-configuration-reference" + }, + { + "title": "Health Checks Configuration Reference", + "path": "services/configuration/checks-configuration-reference" + }, + { + "title": "Service Defaults Configuration Reference", + "href": "connect/config-entries/service-defaults" + } + ] } ] }, @@ -459,11 +510,6 @@ { "title": "Proxy Integration", "path": "connect/proxies/integrate" - }, - { - "title": "Managed (Deprecated)", - "path": "connect/proxies/managed-deprecated", - "hidden": true } ] }, From 06b0867654f89c0bf49b3f329829d3d4951d43a7 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Feb 2023 21:57:04 -0600 Subject: [PATCH 066/421] Backport of Changed titles for services pages to sentence style cap into release/1.15.x (#16479) * backport of commit 239ee7db9b1c4365aee89d8732bab07d4a905791 * backport of commit 129373151659406473ccb903a5d403898049f3a2 --------- Co-authored-by: trujillo-adam --- .../configuration/checks-configuration-reference.mdx | 4 ++-- .../configuration/services-configuration-overview.mdx | 5 +++-- .../configuration/services-configuration-reference.mdx | 5 +++-- .../content/docs/services/discovery/dns-configuration.mdx | 4 ++-- .../content/docs/services/discovery/dns-dynamic-lookups.mdx | 5 +++-- website/content/docs/services/discovery/dns-overview.mdx | 4 ++-- .../content/docs/services/discovery/dns-static-lookups.mdx | 4 ++-- website/content/docs/services/services.mdx | 4 ++-- website/content/docs/services/usage/checks.mdx | 4 ++-- website/content/docs/services/usage/define-services.mdx | 4 ++-- .../content/docs/services/usage/register-services-checks.mdx | 5 +++-- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/website/content/docs/services/configuration/checks-configuration-reference.mdx b/website/content/docs/services/configuration/checks-configuration-reference.mdx index 3159c977a208..fee071de51b0 100644 --- a/website/content/docs/services/configuration/checks-configuration-reference.mdx +++ b/website/content/docs/services/configuration/checks-configuration-reference.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Health Check Configuration Reference +page_title: Health check configuration reference description: -> Use the health checks to direct safety functions, such as removing failing nodes and replacing secondary services. Learn how to configure health checks. --- -# Health Check Configuration Reference +# Health check configuration reference This topic provides configuration reference information for health checks. For information about the different kinds of health checks and guidance on defining them, refer to [Define Health Checks]. diff --git a/website/content/docs/services/configuration/services-configuration-overview.mdx b/website/content/docs/services/configuration/services-configuration-overview.mdx index 55be8df32426..3c01f05ae7ca 100644 --- a/website/content/docs/services/configuration/services-configuration-overview.mdx +++ b/website/content/docs/services/configuration/services-configuration-overview.mdx @@ -1,10 +1,11 @@ --- layout: docs -page_title: Services Configuration Overview +page_title: Services configuration overview description: -> This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. --- -# Services Configuration Overview + +# Services configuration overview This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. diff --git a/website/content/docs/services/configuration/services-configuration-reference.mdx b/website/content/docs/services/configuration/services-configuration-reference.mdx index d3ee69e0373f..95f01e16ff73 100644 --- a/website/content/docs/services/configuration/services-configuration-reference.mdx +++ b/website/content/docs/services/configuration/services-configuration-reference.mdx @@ -1,9 +1,10 @@ --- -page_title: Service Configuration Reference +layout: docs +page_title: Service configuration reference description: Use the service definition to configure and register services to the Consul catalog, including services used as proxies in a Consul service mesh --- -# Services Configuration Reference +# Services configuration reference This topic describes the options you can use to define services for registering them with Consul. Refer to the following topics for usage information: diff --git a/website/content/docs/services/discovery/dns-configuration.mdx b/website/content/docs/services/discovery/dns-configuration.mdx index 7e43e7b75bb2..794be43a206a 100644 --- a/website/content/docs/services/discovery/dns-configuration.mdx +++ b/website/content/docs/services/discovery/dns-configuration.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Configure Consul DNS Behavior +page_title: Configure Consul DNS behavior description: -> Learn how to modify the default DNS behavior so that services and nodes can easily discover other services and nodes in your network. --- -# Configure Consul DNS Behavior +# Configure Consul DNS behavior This topic describes the default behavior of the Consul DNS functionality and how to customize how Consul performs queries. diff --git a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx index d19f41c9ea67..d4c8a3fa9eae 100644 --- a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx +++ b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx @@ -1,11 +1,12 @@ --- layout: docs -page_title: Enable Dynamic DNS Queries +page_title: Enable dynamic DNS queries description: -> Learn how to dynamically query the Consul DNS using prepared queries, which enable robust service and node lookups. --- -# Enable Dynamic DNS Queries +# Enable dynamic DNS aueries + This topic describes how to dynamically query the Consul catalog using prepared queries. Prepared queries are configurations that enable you to register a complex service query and execute it on demand. For information about how to perform standard node and service lookups, refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups). ## Introduction diff --git a/website/content/docs/services/discovery/dns-overview.mdx b/website/content/docs/services/discovery/dns-overview.mdx index 53baac080e77..37eda715de25 100644 --- a/website/content/docs/services/discovery/dns-overview.mdx +++ b/website/content/docs/services/discovery/dns-overview.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: DNS Usage Overview +page_title: DNS usage overview description: >- For service discovery use cases, Domain Name Service (DNS) is the main interface to look up, query, and address Consul nodes and services. Learn how a Consul DNS lookup can help you find services by tag, name, namespace, partition, datacenter, or domain. --- -# DNS Usage Overview +# DNS usage overview This topic provides overview information about how to look up Consul nodes and services using the Consul DNS. diff --git a/website/content/docs/services/discovery/dns-static-lookups.mdx b/website/content/docs/services/discovery/dns-static-lookups.mdx index d95987671ef9..68191104a2a4 100644 --- a/website/content/docs/services/discovery/dns-static-lookups.mdx +++ b/website/content/docs/services/discovery/dns-static-lookups.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Perform Static DNS Queries +page_title: Perform static DNS queries description: -> Learn how to use standard Consul DNS lookup formats to enable service discovery for services and nodes. --- -# Perform Static DNS Queries +# Perform static DNS queries This topic describes how to query the Consul DNS to look up nodes and services registered with Consul. Refer to [Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for information about using prepared queries. ## Introduction diff --git a/website/content/docs/services/services.mdx b/website/content/docs/services/services.mdx index 3a2f21538907..b24562e64e32 100644 --- a/website/content/docs/services/services.mdx +++ b/website/content/docs/services/services.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Services Overview +page_title: Services overview description: >- Learn about services and service discovery workflows and concepts for virtual machine environments. --- -# Services Overview +# Services overview This topic provides overview information about services and how to make them discoverable in Consul when your network operates on virtual machines. If service mesh is enabled in your network, refer to the topics in [Consul Service Mesh](/consul/docs/connect/). If your network is running in Kubernetes, refer to the topics in [Consul on Kubernetes](/consul/docs/k8s). ## Introduction diff --git a/website/content/docs/services/usage/checks.mdx b/website/content/docs/services/usage/checks.mdx index 99d26c722292..8f86b2283975 100644 --- a/website/content/docs/services/usage/checks.mdx +++ b/website/content/docs/services/usage/checks.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Define Health Checks +page_title: Define health checks description: -> Learn how to configure different types of health checks for services you register with Consul. --- -# Define Health Checks +# Define health checks This topic describes how to create different types of health checks for your services. diff --git a/website/content/docs/services/usage/define-services.mdx b/website/content/docs/services/usage/define-services.mdx index 35df236668d2..a4b6eaa15961 100644 --- a/website/content/docs/services/usage/define-services.mdx +++ b/website/content/docs/services/usage/define-services.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Define Services +page_title: Define services description: >- Learn how to define services so that they are discoverable in your network. --- -# Define Services +# Define services This topic describes how to define services so that they can be discovered by other services. Refer to [Services Overview](/consul/docs/services/services) for additional information. diff --git a/website/content/docs/services/usage/register-services-checks.mdx b/website/content/docs/services/usage/register-services-checks.mdx index fca829a00e1c..89c96a1f2e7a 100644 --- a/website/content/docs/services/usage/register-services-checks.mdx +++ b/website/content/docs/services/usage/register-services-checks.mdx @@ -1,11 +1,12 @@ --- layout: docs -page_title: Register Services and Health Checks +page_title: Register services and health checks description: -> Learn how to register services and health checks with Consul agents. --- -# Register Services and Health Checks +# Register services and health checks + This topic describes how to register services and health checks with Consul in networks running on virtual machines (VM). Refer to [Define Services](/consul/usage/services/usage/define-services) and [Define Health Checks](/consul/usage/services/usage/checks) for information about how to define services and health checks. ## Overview From cafe8ee57fcc3fdcb1bc3430f526ffd518bb05e6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 1 Mar 2023 02:22:50 -0600 Subject: [PATCH 067/421] Backport of docs: Consul 1.15.0 and Consul K8s 1.0 release notes into release/1.15.x (#16482) * merge conflict --------- Co-authored-by: david-yu --- .../docs/release-notes/consul-k8s/v0_49_x.mdx | 4 + .../docs/release-notes/consul-k8s/v1_0_x.mdx | 4 + .../docs/release-notes/consul-k8s/v1_1_x.mdx | 56 +++++++++++++ .../docs/release-notes/consul/v1_15_x.mdx | 81 +++++++++++++++++++ website/data/docs-nav-data.json | 8 ++ 5 files changed, 153 insertions(+) create mode 100644 website/content/docs/release-notes/consul-k8s/v1_1_x.mdx create mode 100644 website/content/docs/release-notes/consul/v1_15_x.mdx diff --git a/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx index 57a66e5c3e5e..cb03589e1cfc 100644 --- a/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx @@ -44,3 +44,7 @@ The changelogs for this major release version and any maintenance versions are l ~> **Note:** The following link takes you to the changelogs on the GitHub website. - [0.49.0](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.0) +- [0.49.1](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.1) +- [0.49.2](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.2) +- [0.49.3](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.3) +- [0.49.4](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.4) diff --git a/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx b/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx index f34e2397c27d..b9236898dd51 100644 --- a/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx @@ -61,3 +61,7 @@ The changelogs for this major release version and any maintenance versions are l ~> **Note:** The following link takes you to the changelogs on the GitHub website. - [1.0.0](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.0) +- [1.0.1](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.1) +- [1.0.2](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.2) +- [1.0.3](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.3) +- [1.0.4](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.4) diff --git a/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx new file mode 100644 index 000000000000..6931aecd7055 --- /dev/null +++ b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx @@ -0,0 +1,56 @@ +--- +layout: docs +page_title: 1.1.x +description: >- + Consul on Kubernetes release notes for version 1.1.x +--- + +# Consul on Kubernetes 1.1.0 + +## Release Highlights + +- **Enhanced Envoy Access Logging:** Envoy access logs are now centrally managed via the `accessLogs` field within the ProxyDefaults CRD to allow operators to easily turn on access logs for all proxies within the service mesh. Refer to [Access logs overview](/consul/docs/connect/observability/access-logs) for more information. + +- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. +The new `envoyExtensions` field in the ProxyDefaults and ServiceDefaults CRDs enable built-in Envoy extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions. + +## What's Changed + +- Connect inject now excludes the `openebs` namespace from sidecar injection by default. If you previously had pods in that namespace +that you wanted to be injected, you must now set namespaceSelector as follows: + + ```yaml + connectInject: + namespaceSelector: | + matchExpressions: + - key: "kubernetes.io/metadata.name" + operator: "NotIn" + values: ["kube-system","local-path-storage"] + ``` + +## Supported Software + +~> **Note:** Consul 1.14.x and 1.13.x are not supported. Please refer to [Supported Consul and Kubernetes versions](/consul/docs/k8s/compatibility#supported-consul-and-kubernetes-versions) for more detail on choosing the correct `consul-k8s` version. +- Consul 1.15.x. +- Consul Dataplane v1.1.x. Refer to [Envoy and Consul Dataplane](/consul/docs/connect/proxies/envoy#envoy-and-consul-dataplane) for details about Consul Dataplane versions and the available packaged Envoy version. +- Kubernetes 1.23.x - 1.26.x +- `kubectl` 1.23.x - 1.26.x +- Helm 3.6+ + +## Upgrading + +For detailed information on upgrading, please refer to the [Upgrades page](/consul/docs/k8s/upgrade) + +## Known Issues + +The following issues are known to exist in the v1.1.0 release: + +- Pod Security Standards that are configured for the [Pod Security Admission controller](https://kubernetes.io/blog/2022/08/25/pod-security-admission-stable/) are currently not supported by Consul K8s. OpenShift 4.11.x enables Pod Security Standards on Kubernetes 1.25 [by default](https://connect.redhat.com/en/blog/important-openshift-changes-pod-security-standards) and is also not supported. Support will be added in a future Consul K8s 1.0.x patch release. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +~> **Note:** The following link takes you to the changelogs on the GitHub website. + +- [1.1.0](https://github.com/hashicorp/consul-k8s/releases/tag/v1.1.0) diff --git a/website/content/docs/release-notes/consul/v1_15_x.mdx b/website/content/docs/release-notes/consul/v1_15_x.mdx new file mode 100644 index 000000000000..3877f6ccd7bb --- /dev/null +++ b/website/content/docs/release-notes/consul/v1_15_x.mdx @@ -0,0 +1,81 @@ +--- +layout: docs +page_title: 1.15.x +description: >- + Consul release notes for version 1.15.x +--- + +# Consul 1.15.0 + +## Release Highlights + +- **Enhanced Envoy Access Logging:** Envoy access logs are now centrally managed via config entries and CRDs to allow operators to easily turn on access logs for all proxies within the service mesh. Refer to [Access logs overview](/consul/docs/connect/observability/access-logs) for more information. Additionally, the [Proxy default configuration entry](/consul/docs/connect/config-entries/proxy-defaults) shows you how to enable access logs centrally via the ProxyDefaults config entry or CRD. + +- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. +Current supported extensions include the [Lua](/consul/docs/connect/proxies/envoy-extensions#lua) and [AWS Lambda](/consul/docs/connect/proxies/envoy-extensions#lambda) extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions for Consul service mesh. + +- **API Gateway support on Linux VM runtimes:** You can now deploy Consul API Gateway on Linux VM runtimes. API Gateway is built into Consul and, when deploying on Linux VM runtimes, is not separately installed software. Refer to [API gateway overview](/consul/docs/connect/gateways/api-gateway) for more information on API Gateway specifically for VM. + + ~> **Note:** Support for API Gateway on Linux VM runtimes is considered a "Beta" feature in Consul v1.15.0. HashiCorp expects to change it to a GA feature as part of a v1.15 patch release in the near future. + +- **Limit traffic rates to Consul servers:** You can now configure global RPC rate limits to mitigate the risks to Consul servers when clients send excessive read or write requests to Consul resources. Refer to [Limit traffic rates overview](/consul/docs/agent/limits) for more details on how to use the new troubleshooting commands. + +- **Service-to-service troubleshooting:** Consul includes a built-in tool for troubleshooting communication between services in a service mesh. The `consul troubleshoot` command enables you to validate communication between upstream and downstream Envoy proxies on VM and Kubernetes deployments. Refer to [Service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services) for more details on how to use the new troubleshooting commands. +Refer to [Service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services) for more details on how to use the new troubleshooting commands. + +- **Raft write-ahead log (Experimental):** Consul provides an experimental storage backend called write-ahead log (WAL). WAL implements a traditional log with rotating, append-only log files which resolves a number of performance issues with the current BoltDB storage backend. Refer to [Experimental WAL LogStore backend overview](/consul/docs/agent/wal-logstore) for more details. + + ~> **Note:** The new Raft write-ahead log storage backend is not recommended for production use cases yet, but is ready for testing by the general community. + +## What's Changed + +- ACL errors have now been ehanced to return descriptive errors when the specified resource cannot be found. Other ACL request errors provide more information about when a resource is missing. In addition, errors are now gracefully thrown when interacting with the ACL system before the ACL system been bootstrapped. + - The Delete Token/Policy/AuthMethod/Role/BindingRule endpoints now return 404 when the resource cannot be found. The new error format is as follows: + + ```log hideClipboard + Requested * does not exist: ACL not found", "* not found in namespace $NAMESPACE: ACL not found` + ``` + + - The Read Token/Policy/Role endpoints now return 404 when the resource cannot be found. The new error format is as follows: + + ```log hideClipboard + Cannot find * to delete + ``` + + - The Logout endpoint now returns a 401 error when the supplied token cannot be found. The new error format is as follows: + + ```log hideClipboard + Supplied token does not exist + ``` + + - The Token Self endpoint now returns 404 when the token cannot be found. The new error format is as follows: + + ```log hideClipboard + Supplied token does not exist + ``` + +- Consul v1.15.0 formally removes all uses of legacy ACLs and ACL policies from Consul. The legacy ACL system was deprecated in Consul v1.4.0 and removed in Consul v1.11.0. The documentation for the new ACL system can be found [here](/consul/docs/v1.14.x/security/acl). For information on how to migrate to the new ACL System, please read the [Migrate Legacy ACL Tokens tutorial](/consul/tutorials/security-operations/access-control-token-migration). +- The following agent flags are now deprecated: `-join`, `-join-wan`, `start_join`, and `start_join_wan`. These options are now aliases of `-retry-join`, `-retry-join-wan`, `retry_join`, and `retry_join_wan`, respectively. +- A `peer` field has been added to ServiceDefaults upstream overrides to make it possible to apply upstream overrides only to peer services. Prior to this change, overrides would be applied based on matching the namespace and name fields only, which means users could not have different configuration for local versus peer services. With this change, peer upstreams are only affected if the peer field matches the destination peer name. +- If you run the `consul connect envoy` command with an incompatible Envoy version, Consul will now error and exit. To ignore this check, use flag `--ignore-envoy-compatibility`. +- Ingress Gateway upstream clusters will have empty `outlier_detection` if passive health check is unspecified. + +## Upgrading + +For more detailed information, please refer to the [upgrade details page](/consul/docs/upgrading/upgrade-specific#consul-1-15-0) and the changelogs. + +## Known Issues + +The following issues are known to exist in the v1.15.0 release: + +- For v1.15.0, there is a known issue where `consul acl token read -self` requires an `-accessor-id`. This is resolved in the uppcoming Consul v1.15.1 patch release. + +- For v1.15.0, there is a known issue where search filters produced errors and resulted in lists not showing full results until being interacted with. This is resolved in the upcoming Consul v1.15.1 patch release. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +~> **Note:** These links take you to the changelogs on the GitHub website. + +- [1.15.0](https://github.com/hashicorp/consul/releases/tag/v1.15.0) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 9b53bd05117d..1c2646447407 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -145,6 +145,10 @@ { "title": "Consul", "routes": [ + { + "title": "v1.15.x", + "path": "release-notes/consul/v1_15_x" + }, { "title": "v1.14.x", "path": "release-notes/consul/v1_14_x" @@ -174,6 +178,10 @@ { "title": "Consul K8s", "routes": [ + { + "title": "v1.1.x", + "path": "release-notes/consul-k8s/v1_1_x" + }, { "title": "v1.0.x", "path": "release-notes/consul-k8s/v1_0_x" From 781a2559d8e9a26d7890f4ddd080022882c34667 Mon Sep 17 00:00:00 2001 From: malizz Date: Wed, 1 Mar 2023 10:31:10 -0800 Subject: [PATCH 068/421] update changelog and version (#16427) * update changelog and version * fix changelog formatting --- CHANGELOG.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ version/VERSION | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 975050ecf3e7..1a9c47514261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,85 @@ +## 1.15.0 (February 23, 2023) + +BREAKING CHANGES: + +* acl errors: Delete and get requests now return descriptive errors when the specified resource cannot be found. Other ACL request errors provide more information about when a resource is missing. Add error for when the ACL system has not been bootstrapped. + + Delete Token/Policy/AuthMethod/Role/BindingRule endpoints now return 404 when the resource cannot be found. + - New error formats: "Requested * does not exist: ACL not found", "* not found in namespace $NAMESPACE: ACL not found" + + Read Token/Policy/Role endpoints now return 404 when the resource cannot be found. + - New error format: "Cannot find * to delete" + + Logout now returns a 401 error when the supplied token cannot be found + - New error format: "Supplied token does not exist" + + Token Self endpoint now returns 404 when the token cannot be found. + - New error format: "Supplied token does not exist" [[GH-16105](https://github.com/hashicorp/consul/issues/16105)] +* acl: remove all acl migration functionality and references to the legacy acl system. [[GH-15947](https://github.com/hashicorp/consul/issues/15947)] +* acl: remove all functionality and references for legacy acl policies. [[GH-15922](https://github.com/hashicorp/consul/issues/15922)] +* config: Deprecate `-join`, `-join-wan`, `start_join`, and `start_join_wan`. +These options are now aliases of `-retry-join`, `-retry-join-wan`, `retry_join`, and `retry_join_wan`, respectively. [[GH-15598](https://github.com/hashicorp/consul/issues/15598)] +* connect: Add `peer` field to service-defaults upstream overrides. The addition of this field makes it possible to apply upstream overrides only to peer services. Prior to this change, overrides would be applied based on matching the `namespace` and `name` fields only, which means users could not have different configuration for local versus peer services. With this change, peer upstreams are only affected if the `peer` field matches the destination peer name. [[GH-15956](https://github.com/hashicorp/consul/issues/15956)] +* connect: Consul will now error and exit when using the `consul connect envoy` command if the Envoy version is incompatible. To ignore this check use flag `--ignore-envoy-compatibility` [[GH-15818](https://github.com/hashicorp/consul/issues/15818)] +* extensions: Refactor Lambda integration to get configured with the Envoy extensions field on service-defaults configuration entries. [[GH-15817](https://github.com/hashicorp/consul/issues/15817)] +* ingress-gateway: upstream cluster will have empty outlier_detection if passive health check is unspecified [[GH-15614](https://github.com/hashicorp/consul/issues/15614)] +* xds: Remove the `connect.enable_serverless_plugin` agent configuration option. Now +Lambda integration is enabled by default. [[GH-15710](https://github.com/hashicorp/consul/issues/15710)] + +SECURITY: + +* Upgrade to use Go 1.20.1. +This resolves vulnerabilities [CVE-2022-41724](https://go.dev/issue/58001) in `crypto/tls` and [CVE-2022-41723](https://go.dev/issue/57855) in `net/http`. [[GH-16263](https://github.com/hashicorp/consul/issues/16263)] + +FEATURES: + +* **API Gateway (Beta)** This version adds support for API gateway on VMs. API gateway provides a highly-configurable ingress for requests coming into a Consul network. For more information, refer to the [API gateway](https://developer.hashicorp.com/consul/docs/connect/gateways/api-gateway) documentation. [[GH-16369](https://github.com/hashicorp/consul/issues/16369)] +* acl: Add new `acl.tokens.config_file_registration` config field which specifies the token used +to register services and checks that are defined in config files. [[GH-15828](https://github.com/hashicorp/consul/issues/15828)] +* acl: anonymous token is logged as 'anonymous token' instead of its accessor ID [[GH-15884](https://github.com/hashicorp/consul/issues/15884)] +* cli: adds new CLI commands `consul troubleshoot upstreams` and `consul troubleshoot proxy` to troubleshoot Consul's service mesh configuration and network issues. [[GH-16284](https://github.com/hashicorp/consul/issues/16284)] +* command: Adds the `operator usage instances` subcommand for displaying total services, connect service instances and billable service instances in the local datacenter or globally. [[GH-16205](https://github.com/hashicorp/consul/issues/16205)] +* config-entry(ingress-gateway): support outlier detection (passive health check) for upstream cluster [[GH-15614](https://github.com/hashicorp/consul/issues/15614)] +* connect: adds support for Envoy [access logging](https://developer.hashicorp.com/consul/docs/connect/observability/access-logs). Access logging can be enabled using the [`proxy-defaults`](https://developer.hashicorp.com/consul/docs/connect/config-entries/proxy-defaults#accesslogs) config entry. [[GH-15864](https://github.com/hashicorp/consul/issues/15864)] +* xds: Add a built-in Envoy extension that inserts Lua HTTP filters. [[GH-15906](https://github.com/hashicorp/consul/issues/15906)] +* xds: Insert originator service identity into Envoy's dynamic metadata under the `consul` namespace. [[GH-15906](https://github.com/hashicorp/consul/issues/15906)] + +IMPROVEMENTS: + +* connect: for early awareness of Envoy incompatibilities, when using the `consul connect envoy` command the Envoy version will now be checked for compatibility. If incompatible Consul will error and exit. [[GH-15818](https://github.com/hashicorp/consul/issues/15818)] +* grpc: client agents will switch server on error, and automatically retry on `RESOURCE_EXHAUSTED` responses [[GH-15892](https://github.com/hashicorp/consul/issues/15892)] +* raft: add an operator api endpoint and a command to initiate raft leadership transfer. [[GH-14132](https://github.com/hashicorp/consul/issues/14132)] +* acl: Added option to allow for an operator-generated bootstrap token to be passed to the `acl bootstrap` command. [[GH-14437](https://github.com/hashicorp/consul/issues/14437)] +* agent: Give better error when client specifies wrong datacenter when auto-encrypt is enabled. [[GH-14832](https://github.com/hashicorp/consul/issues/14832)] +* api: updated the go module directive to 1.18. [[GH-15297](https://github.com/hashicorp/consul/issues/15297)] +* ca: support Vault agent auto-auth config for Vault CA provider using AWS/GCP authentication. [[GH-15970](https://github.com/hashicorp/consul/issues/15970)] +* cli: always use name "global" for proxy-defaults config entries [[GH-14833](https://github.com/hashicorp/consul/issues/14833)] +* cli: connect envoy command errors if grpc ports are not open [[GH-15794](https://github.com/hashicorp/consul/issues/15794)] +* client: add support for RemoveEmptyTags in Prepared Queries templates. [[GH-14244](https://github.com/hashicorp/consul/issues/14244)] +* connect: Warn if ACLs are enabled but a token is not provided to envoy [[GH-15967](https://github.com/hashicorp/consul/issues/15967)] +* container: Upgrade container image to use to Alpine 3.17. [[GH-16358](https://github.com/hashicorp/consul/issues/16358)] +* dns: support RFC 2782 SRV lookups for prepared queries using format `_._tcp.query[.].`. [[GH-14465](https://github.com/hashicorp/consul/issues/14465)] +* ingress-gateways: Don't log error when gateway is registered without a config entry [[GH-15001](https://github.com/hashicorp/consul/issues/15001)] +* licensing: **(Enterprise Only)** Consul Enterprise non-terminating production licenses do not degrade or terminate Consul upon expiration. They will only fail when trying to upgrade to a newer version of Consul. Evaluation licenses still terminate. +* raft: Added experimental `wal` backend for log storage. [[GH-16176](https://github.com/hashicorp/consul/issues/16176)] +* sdk: updated the go module directive to 1.18. [[GH-15297](https://github.com/hashicorp/consul/issues/15297)] +* telemetry: Added a `consul.xds.server.streamsUnauthenticated` metric to track +the number of active xDS streams handled by the server that are unauthenticated +because ACLs are not enabled or ACL tokens were missing. [[GH-15967](https://github.com/hashicorp/consul/issues/15967)] +* ui: Update sidebar width to 280px [[GH-16204](https://github.com/hashicorp/consul/issues/16204)] +* ui: update Ember version to 3.27; [[GH-16227](https://github.com/hashicorp/consul/issues/16227)] + +DEPRECATIONS: + +* acl: Deprecate the `token` query parameter and warn when it is used for authentication. [[GH-16009](https://github.com/hashicorp/consul/issues/16009)] +* cli: The `-id` flag on acl token operations has been changed to `-accessor-id` for clarity in documentation. The `-id` flag will continue to work, but operators should use `-accessor-id` in the future. [[GH-16044](https://github.com/hashicorp/consul/issues/16044)] + +BUG FIXES: + +* agent configuration: Fix issue of using unix socket when https is used. [[GH-16301](https://github.com/hashicorp/consul/issues/16301)] +* cache: refactor agent cache fetching to prevent unnecessary fetches on error [[GH-14956](https://github.com/hashicorp/consul/issues/14956)] +* cli: fatal error if config file does not have HCL or JSON extension, instead of warn and skip [[GH-15107](https://github.com/hashicorp/consul/issues/15107)] +* cli: fix ACL token processing unexpected precedence [[GH-15274](https://github.com/hashicorp/consul/issues/15274)] +* peering: Fix bug where services were incorrectly imported as connect-enabled. [[GH-16339](https://github.com/hashicorp/consul/issues/16339)] +* peering: Fix issue where mesh gateways would use the wrong address when contacting a remote peer with the same datacenter name. [[GH-16257](https://github.com/hashicorp/consul/issues/16257)] +* peering: Fix issue where secondary wan-federated datacenters could not be used as peering acceptors. [[GH-16230](https://github.com/hashicorp/consul/issues/16230)] + ## 1.14.4 (January 26, 2023) BREAKING CHANGES: diff --git a/version/VERSION b/version/VERSION index 0dec25d15b37..bf222ecb47b4 100644 --- a/version/VERSION +++ b/version/VERSION @@ -1 +1 @@ -1.15.0-dev \ No newline at end of file +1.15.1-dev \ No newline at end of file From 5a79bfdb6d40d2edec54f22b1baf9ea050195ef3 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 1 Mar 2023 13:53:11 -0600 Subject: [PATCH 069/421] Backport of update services nav titles into release/1.15.x (#16488) * backport of commit 6f5afe38f56143a3a83ef9b3a8a59ad2f741dfc8 * backport of commit dca8a4a34c5843703633a083fa03b4330876ccf7 --------- Co-authored-by: trujillo-adam --- website/data/docs-nav-data.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 1c2646447407..f66afc94c5f1 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -314,43 +314,43 @@ "title": "Services", "routes": [ { - "title": "Services Overview", + "title": "Overview", "path": "services/services" }, { - "title": "Define and Register", + "title": "Usage", "routes": [ { - "title": "Define Services", + "title": "Define services", "path": "services/usage/define-services" }, { - "title": "Define Health Checks", + "title": "Define health checks", "path": "services/usage/checks" }, { - "title": "Register Services and Health Checks", + "title": "Register services and health checks", "path": "services/usage/register-services-checks" } ] }, { - "title": "Service Discovery", + "title": "Discover services with DNS", "routes": [ { - "title": "DNS Usage Overview", + "title": "Overview", "path": "services/discovery/dns-overview" }, { - "title": "Configure Consul DNS Behavior", + "title": "Configure DNS behavior", "path": "services/discovery/dns-configuration" }, { - "title": "Perform Static DNS Lookups", + "title": "Perform static DNS lookups", "path": "services/discovery/dns-static-lookups" }, { - "title": "Enable Dynamic DNS Lookups", + "title": "Enable dynamic DNS lookups", "path": "services/discovery/dns-dynamic-lookups" } ] @@ -359,19 +359,19 @@ "title": "Configuration", "routes": [ { - "title": "Services Configuration Overview", + "title": "Overview", "path": "services/configuration/services-configuration-overview" }, { - "title": "Services Configuration Reference", + "title": "Services", "path": "services/configuration/services-configuration-reference" }, { - "title": "Health Checks Configuration Reference", + "title": "Health checks", "path": "services/configuration/checks-configuration-reference" }, { - "title": "Service Defaults Configuration Reference", + "title": "Service defaults", "href": "connect/config-entries/service-defaults" } ] From cf743a36b71b1152ada1c8161f9d1171ee837b86 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 1 Mar 2023 14:33:50 -0600 Subject: [PATCH 070/421] Backport of fix (cli): return error msg if acl policy not found into release/1.15.x (#16486) * backport of commit 77011a075ef8ed7353ef5bfb9e740e26875f0229 * backport of commit 8efa879d7d308cf63c76f356aaf27de8f0a1521a * backport of commit 45944e4aa1b9a0999088cbfd94de6376f3b41144 --------- Co-authored-by: cskh --- .changelog/16485.txt | 3 +++ command/acl/policy/read/policy_read.go | 5 +++++ command/acl/policy/read/policy_read_test.go | 11 +++++++++++ 3 files changed, 19 insertions(+) create mode 100644 .changelog/16485.txt diff --git a/.changelog/16485.txt b/.changelog/16485.txt new file mode 100644 index 000000000000..7e1938b00ea0 --- /dev/null +++ b/.changelog/16485.txt @@ -0,0 +1,3 @@ +```release-note:bug +cli: fix panic read non-existent acl policy +``` diff --git a/command/acl/policy/read/policy_read.go b/command/acl/policy/read/policy_read.go index 455f5e5f7d67..57e9397b2a47 100644 --- a/command/acl/policy/read/policy_read.go +++ b/command/acl/policy/read/policy_read.go @@ -92,6 +92,11 @@ func (c *cmd) Run(args []string) int { return 1 } + if pol == nil { + c.UI.Error(fmt.Sprintf("Error policy not found: %s", c.policyName)) + return 1 + } + formatter, err := policy.NewFormatter(c.format, c.showMeta) if err != nil { c.UI.Error(err.Error()) diff --git a/command/acl/policy/read/policy_read_test.go b/command/acl/policy/read/policy_read_test.go index af5bf1c843ff..bd8d99dd8374 100644 --- a/command/acl/policy/read/policy_read_test.go +++ b/command/acl/policy/read/policy_read_test.go @@ -82,6 +82,17 @@ func TestPolicyReadCommand(t *testing.T) { output = ui.OutputWriter.String() assert.Contains(t, output, fmt.Sprintf("test-policy")) assert.Contains(t, output, policy.ID) + + // Test querying non-existent policy + argsName = []string{ + "-http-addr=" + a.HTTPAddr(), + "-token=root", + "-name=test-policy-not-exist", + } + + cmd = New(ui) + code = cmd.Run(argsName) + assert.Equal(t, code, 1) } func TestPolicyReadCommand_JSON(t *testing.T) { From 0b85dc39a2e8834371c4829375eb6c205a15fbeb Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 1 Mar 2023 16:22:40 -0600 Subject: [PATCH 071/421] Backport of docs: Update release notes with Envoy compat issue into release/1.15.x (#16496) * backport of commit f189a8203cdca3dd393a38699bdf9d934f9426f6 * backport of commit 82d9805943d99223c4a3fcca15492792b7bc4c83 * backport of commit f1d95252a8b43d36f5cd226ce04e88ad8e071066 * backport of commit 1c8d737beedb3d365926e2384b3b4a948971bb88 --------- Co-authored-by: David Yu --- .../content/docs/release-notes/consul/v1_15_x.mdx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/website/content/docs/release-notes/consul/v1_15_x.mdx b/website/content/docs/release-notes/consul/v1_15_x.mdx index 3877f6ccd7bb..5d8ffd9dc21c 100644 --- a/website/content/docs/release-notes/consul/v1_15_x.mdx +++ b/website/content/docs/release-notes/consul/v1_15_x.mdx @@ -68,6 +68,18 @@ For more detailed information, please refer to the [upgrade details page](/consu The following issues are known to exist in the v1.15.0 release: +- For v1.15.0, Consul is reporting newer releases of Envoy (for example, v1.25.1) as not supported, even though these versions are listed as valid in the [Envoy compatilibity matrix](/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent). The following error would result for newer versions of Envoy: + + ```log hideClipboard + Envoy version 1.25.1 is not supported. If there is a reason you need to use this version of envoy use the ignore-envoy-compatibility flag. Using an unsupported version of Envoy is not recommended and your experience may vary. + ``` + + The workaround to resolve this issue until Consul v1.15.1 would be to run the client agents with the new `ingore-envoy-compatiblity` flag: + + ```shell-session + $ consul connect envoy --ignore-envoy-compatibility + ``` + - For v1.15.0, there is a known issue where `consul acl token read -self` requires an `-accessor-id`. This is resolved in the uppcoming Consul v1.15.1 patch release. - For v1.15.0, there is a known issue where search filters produced errors and resulted in lists not showing full results until being interacted with. This is resolved in the upcoming Consul v1.15.1 patch release. From 381760b8c29bc0864120eb53cbf4a0a11a52d2ec Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Wed, 1 Mar 2023 18:10:36 -0500 Subject: [PATCH 072/421] [OSS] connect: Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 (#16274) (#16491) * Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 Co-authored-by: Curt Bushko --- .changelog/16274.txt | 3 +++ .circleci/config.yml | 8 ++++---- envoyextensions/xdscommon/envoy_versioning_test.go | 7 ++++--- envoyextensions/xdscommon/proxysupport.go | 6 +++--- website/content/docs/connect/proxies/envoy.mdx | 6 +++--- 5 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 .changelog/16274.txt diff --git a/.changelog/16274.txt b/.changelog/16274.txt new file mode 100644 index 000000000000..983d33b19599 --- /dev/null +++ b/.changelog/16274.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 +``` diff --git a/.circleci/config.yml b/.circleci/config.yml index d50ddb1fa72a..46ebfe240e91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,10 +23,10 @@ references: BASH_ENV: .circleci/bash_env.sh GO_VERSION: 1.20.1 envoy-versions: &supported_envoy_versions - - &default_envoy_version "1.21.5" - - "1.22.5" - - "1.23.2" - - "1.24.0" + - &default_envoy_version "1.22.7" + - "1.23.4" + - "1.24.2" + - "1.25.1" nomad-versions: &supported_nomad_versions - &default_nomad_version "1.3.3" - "1.2.10" diff --git a/envoyextensions/xdscommon/envoy_versioning_test.go b/envoyextensions/xdscommon/envoy_versioning_test.go index 833e3014ebee..e20a2ca8ceef 100644 --- a/envoyextensions/xdscommon/envoy_versioning_test.go +++ b/envoyextensions/xdscommon/envoy_versioning_test.go @@ -121,6 +121,7 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) { "1.18.6": {expectErr: "Envoy 1.18.6 " + errTooOld}, "1.19.5": {expectErr: "Envoy 1.19.5 " + errTooOld}, "1.20.7": {expectErr: "Envoy 1.20.7 " + errTooOld}, + "1.21.5": {expectErr: "Envoy 1.21.5 " + errTooOld}, } // Insert a bunch of valid versions. @@ -135,10 +136,10 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) { } */ for _, v := range []string{ - "1.21.0", "1.21.1", "1.21.2", "1.21.3", "1.21.4", "1.21.5", "1.22.0", "1.22.1", "1.22.2", "1.22.3", "1.22.4", "1.22.5", - "1.23.0", "1.23.1", "1.23.2", - "1.24.0", + "1.23.0", "1.23.1", "1.23.2", "1.23.3", "1.23.4", + "1.24.0", "1.24.1", "1.24.2", + "1.25.0", "1.25.1", } { cases[v] = testcase{expect: SupportedProxyFeatures{}} } diff --git a/envoyextensions/xdscommon/proxysupport.go b/envoyextensions/xdscommon/proxysupport.go index 963e0dba0c2a..bedc0608bfd3 100644 --- a/envoyextensions/xdscommon/proxysupport.go +++ b/envoyextensions/xdscommon/proxysupport.go @@ -9,10 +9,10 @@ import "strings" // // see: https://www.consul.io/docs/connect/proxies/envoy#supported-versions var EnvoyVersions = []string{ - "1.24.0", - "1.23.2", + "1.25.1", + "1.24.2", + "1.23.4", "1.22.5", - "1.21.5", } // UnsupportedEnvoyVersions lists any unsupported Envoy versions (mainly minor versions) that fall diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 61ea0bf738c3..bb27db3b2a31 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -45,7 +45,7 @@ Consul supports **four major Envoy releases** at the beginning of each major Con ### Envoy and Consul Dataplane -The Consul dataplane component was introduced in Consul v1.14 as a way to manage Envoy proxies without the use of Consul clients. Each new minor version of Consul is released with a new minor version of Consul dataplane, which packages both Envoy and the `consul-dataplane` binary in a single container image. For backwards compatability reasons, each new minor version of Consul will also support the previous minor version of Consul dataplane to allow for seamless upgrades. In addition, each minor version of Consul will support the next minor version of Consul dataplane to allow for extended dataplane support via newer versions of Envoy. +The Consul dataplane component was introduced in Consul v1.14 as a way to manage Envoy proxies without the use of Consul clients. Each new minor version of Consul is released with a new minor version of Consul dataplane, which packages both Envoy and the `consul-dataplane` binary in a single container image. For backwards compatability reasons, each new minor version of Consul will also support the previous minor version of Consul dataplane to allow for seamless upgrades. In addition, each minor version of Consul will support the next minor version of Consul dataplane to allow for extended dataplane support via newer versions of Envoy. | Consul Version | Consul Dataplane Version (Bundled Envoy Version) | | ------------------- | ------------------------------------------------- | @@ -124,7 +124,7 @@ Connect to a local Consul client agent and run the [`consul connect envoy` comma If you experience issues when bootstrapping Envoy proxies from the CLI, use the `-enable-config-gen-logging` flag to enable debug message logging. These logs can -help you troubleshoot issues that occur during the bootstrapping process. +help you troubleshoot issues that occur during the bootstrapping process. For more information about available flags and parameters, refer to the [`consul connect envoy CLI` reference](/consul/commands/connect/envoy). @@ -336,7 +336,7 @@ defaults that are inherited by all services. - `local_idle_timeout_ms` - In milliseconds, the idle timeout for HTTP requests to the local application instance. Applies to HTTP based protocols only. If not - specified, inherits the Envoy default for route idle timeouts (15s). A value of 0 + specified, inherits the Envoy default for route idle timeouts (15s). A value of 0 disables request timeouts. - `max_inbound_connections` - The maximum number of concurrent inbound connections From 8e0fd0e605319e583047e67ca4637ad4a8be7975 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 2 Mar 2023 12:02:25 -0600 Subject: [PATCH 073/421] Backport of Improve ux to help users avoid overwriting fields of ACL tokens, roles and policies into release/1.15.x (#16489) * backport of commit 5a10c732e25e10117169e8b63860d4621345f8e9 * backport of commit dbf3a1941057d25894a28ff4ac965d6dec523f74 * backport of commit 29a84c48ebb6321c5034c2f39e5b616eb54b91e3 * backport of commit 63d2fb4d913698df9352aa0788e2a84fe07bdbb0 * backport of commit f4048866f1137dbfd6b3dea0d93bedcbadc07fb0 * backport of commit e7c0a45492ede85c5745351f35df498765c1c54e * backport of commit 5a37d37179b104e9e60b11474261b86c900a0406 --------- Co-authored-by: Ronald Ekambi Co-authored-by: Ronald --- .changelog/16288.txt | 8 ++ command/acl/token/update/token_update.go | 91 +++++++++++++++---- command/acl/token/update/token_update_test.go | 89 ++++++++++++++++++ website/content/commands/acl/token/update.mdx | 30 ++++-- 4 files changed, 193 insertions(+), 25 deletions(-) create mode 100644 .changelog/16288.txt diff --git a/.changelog/16288.txt b/.changelog/16288.txt new file mode 100644 index 000000000000..5e2820ec27d5 --- /dev/null +++ b/.changelog/16288.txt @@ -0,0 +1,8 @@ +```release-note:deprecation +cli: Deprecate the `-merge-policies` and `-merge-roles` flags from the `consul token update` command in favor of: `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id`. +``` + +```release-note:improvement +cli: added `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id` flags to the `consul token update` command. +These flags allow updates to a token's policies/roles without having to override them completely. +``` \ No newline at end of file diff --git a/command/acl/token/update/token_update.go b/command/acl/token/update/token_update.go index 4f168e927b89..6cb529edd45f 100644 --- a/command/acl/token/update/token_update.go +++ b/command/acl/token/update/token_update.go @@ -27,30 +27,31 @@ type cmd struct { tokenAccessorID string policyIDs []string + appendPolicyIDs []string policyNames []string + appendPolicyNames []string roleIDs []string + appendRoleIDs []string roleNames []string + appendRoleNames []string serviceIdents []string nodeIdents []string description string - mergePolicies bool - mergeRoles bool mergeServiceIdents bool mergeNodeIdents bool showMeta bool format string - tokenID string // DEPRECATED + // DEPRECATED + mergeRoles bool + mergePolicies bool + tokenID string } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.showMeta, "meta", false, "Indicates that token metadata such "+ "as the content hash and raft indices should be shown for each entry") - c.flags.BoolVar(&c.mergePolicies, "merge-policies", false, "Merge the new policies "+ - "with the existing policies") - c.flags.BoolVar(&c.mergeRoles, "merge-roles", false, "Merge the new roles "+ - "with the existing roles") c.flags.BoolVar(&c.mergeServiceIdents, "merge-service-identities", false, "Merge the new service identities "+ "with the existing service identities") c.flags.BoolVar(&c.mergeNodeIdents, "merge-node-identities", false, "Merge the new node identities "+ @@ -60,13 +61,21 @@ func (c *cmd) init() { "matches multiple token Accessor IDs") c.flags.StringVar(&c.description, "description", "", "A description of the token") c.flags.Var((*flags.AppendSliceValue)(&c.policyIDs), "policy-id", "ID of a "+ - "policy to use for this token. May be specified multiple times") + "policy to use for this token. Overwrites existing policies. May be specified multiple times") + c.flags.Var((*flags.AppendSliceValue)(&c.appendPolicyIDs), "append-policy-id", "ID of a "+ + "policy to use for this token. The token retains existing policies. May be specified multiple times") c.flags.Var((*flags.AppendSliceValue)(&c.policyNames), "policy-name", "Name of a "+ - "policy to use for this token. May be specified multiple times") + "policy to use for this token. Overwrites existing policies. May be specified multiple times") + c.flags.Var((*flags.AppendSliceValue)(&c.appendPolicyNames), "append-policy-name", "Name of a "+ + "policy to add to this token. The token retains existing policies. May be specified multiple times") c.flags.Var((*flags.AppendSliceValue)(&c.roleIDs), "role-id", "ID of a "+ - "role to use for this token. May be specified multiple times") + "role to use for this token. Overwrites existing roles. May be specified multiple times") c.flags.Var((*flags.AppendSliceValue)(&c.roleNames), "role-name", "Name of a "+ - "role to use for this token. May be specified multiple times") + "role to use for this token. Overwrites existing roles. May be specified multiple times") + c.flags.Var((*flags.AppendSliceValue)(&c.appendRoleIDs), "append-role-id", "ID of a "+ + "role to add to this token. The token retains existing roles. May be specified multiple times") + c.flags.Var((*flags.AppendSliceValue)(&c.appendRoleNames), "append-role-name", "Name of a "+ + "role to add to this token. The token retains existing roles. May be specified multiple times") c.flags.Var((*flags.AppendSliceValue)(&c.serviceIdents), "service-identity", "Name of a "+ "service identity to use for this token. May be specified multiple times. Format is "+ "the SERVICENAME or SERVICENAME:DATACENTER1,DATACENTER2,...") @@ -87,8 +96,11 @@ func (c *cmd) init() { c.help = flags.Usage(help, c.flags) // Deprecations - c.flags.StringVar(&c.tokenID, "id", "", - "DEPRECATED. Use -accessor-id instead.") + c.flags.StringVar(&c.tokenID, "id", "", "DEPRECATED. Use -accessor-id instead.") + c.flags.BoolVar(&c.mergePolicies, "merge-policies", false, "DEPRECATED. "+ + "Use -append-policy-id or -append-policy-name instead.") + c.flags.BoolVar(&c.mergeRoles, "merge-roles", false, "DEPRECATED. "+ + "Use -append-role-id or -append-role-name instead.") } func (c *cmd) Run(args []string) int { @@ -148,6 +160,9 @@ func (c *cmd) Run(args []string) int { } if c.mergePolicies { + c.UI.Warn("merge-policies is deprecated and will be removed in a future Consul version. " + + "Use `append-policy-name` or `append-policy-id` instead.") + for _, policyName := range c.policyNames { found := false for _, link := range t.Policies { @@ -184,15 +199,33 @@ func (c *cmd) Run(args []string) int { } } } else { - t.Policies = nil - for _, policyName := range c.policyNames { + hasAddPolicyFields := len(c.appendPolicyNames) > 0 || len(c.appendPolicyIDs) > 0 + hasPolicyFields := len(c.policyIDs) > 0 || len(c.policyNames) > 0 + + if hasPolicyFields && hasAddPolicyFields { + c.UI.Error("Cannot combine the use of policy-id/policy-name flags with append- variants. " + + "To set or overwrite existing policies, use -policy-id or -policy-name. " + + "To append to existing policies, use -append-policy-id or -append-policy-name.") + return 1 + } + + policyIDs := c.appendPolicyIDs + policyNames := c.appendPolicyNames + + if hasPolicyFields { + policyIDs = c.policyIDs + policyNames = c.policyNames + t.Policies = nil + } + + for _, policyName := range policyNames { // We could resolve names to IDs here but there isn't any reason why its would be better // than allowing the agent to do it. t.Policies = append(t.Policies, &api.ACLTokenPolicyLink{Name: policyName}) } - for _, policyID := range c.policyIDs { + for _, policyID := range policyIDs { policyID, err := acl.GetPolicyIDFromPartial(client, policyID) if err != nil { c.UI.Error(fmt.Sprintf("Error resolving policy ID %s: %v", policyID, err)) @@ -203,6 +236,9 @@ func (c *cmd) Run(args []string) int { } if c.mergeRoles { + c.UI.Warn("merge-roles is deprecated and will be removed in a future Consul version. " + + "Use `append-role-name` or `append-role-id` instead.") + for _, roleName := range c.roleNames { found := false for _, link := range t.Roles { @@ -239,15 +275,32 @@ func (c *cmd) Run(args []string) int { } } } else { - t.Roles = nil + hasAddRoleFields := len(c.appendRoleNames) > 0 || len(c.appendRoleIDs) > 0 + hasRoleFields := len(c.roleIDs) > 0 || len(c.roleNames) > 0 - for _, roleName := range c.roleNames { + if hasRoleFields && hasAddRoleFields { + c.UI.Error("Cannot combine the use of role-id/role-name flags with append- variants. " + + "To set or overwrite existing roles, use -role-id or -role-name. " + + "To append to existing roles, use -append-role-id or -append-role-name.") + return 1 + } + + roleNames := c.appendRoleNames + roleIDs := c.appendRoleIDs + + if hasRoleFields { + roleNames = c.roleNames + roleIDs = c.roleIDs + t.Roles = nil + } + + for _, roleName := range roleNames { // We could resolve names to IDs here but there isn't any reason why its would be better // than allowing the agent to do it. t.Roles = append(t.Roles, &api.ACLTokenRoleLink{Name: roleName}) } - for _, roleID := range c.roleIDs { + for _, roleID := range roleIDs { roleID, err := acl.GetRoleIDFromPartial(client, roleID) if err != nil { c.UI.Error(fmt.Sprintf("Error resolving role ID %s: %v", roleID, err)) diff --git a/command/acl/token/update/token_update_test.go b/command/acl/token/update/token_update_test.go index 541d1e22539f..bd11d1d8e3f6 100644 --- a/command/acl/token/update/token_update_test.go +++ b/command/acl/token/update/token_update_test.go @@ -148,6 +148,95 @@ func TestTokenUpdateCommand(t *testing.T) { }) } +func TestTokenUpdateCommandWithAppend(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + a := agent.NewTestAgent(t, ` + primary_datacenter = "dc1" + acl { + enabled = true + tokens { + initial_management = "root" + } + }`) + + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc1") + + // Create a policy + client := a.Client() + + policy, _, err := client.ACL().PolicyCreate( + &api.ACLPolicy{Name: "test-policy"}, + &api.WriteOptions{Token: "root"}, + ) + require.NoError(t, err) + + // create a token + token, _, err := client.ACL().TokenCreate( + &api.ACLToken{Description: "test", Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}}, + &api.WriteOptions{Token: "root"}, + ) + require.NoError(t, err) + + //secondary policy + secondPolicy, _, policyErr := client.ACL().PolicyCreate( + &api.ACLPolicy{Name: "secondary-policy"}, + &api.WriteOptions{Token: "root"}, + ) + require.NoError(t, policyErr) + + //third policy + thirdPolicy, _, policyErr := client.ACL().PolicyCreate( + &api.ACLPolicy{Name: "third-policy"}, + &api.WriteOptions{Token: "root"}, + ) + require.NoError(t, policyErr) + + run := func(t *testing.T, args []string) *api.ACLToken { + ui := cli.NewMockUi() + cmd := New(ui) + + code := cmd.Run(append(args, "-format=json")) + require.Equal(t, 0, code) + require.Empty(t, ui.ErrorWriter.String()) + + var token api.ACLToken + require.NoError(t, json.Unmarshal(ui.OutputWriter.Bytes(), &token)) + return &token + } + + // update with append-policy-name + t.Run("append-policy-name", func(t *testing.T) { + token := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-policy-name=" + secondPolicy.Name, + "-description=test token", + }) + + require.Len(t, token.Policies, 2) + }) + + // update with append-policy-id + t.Run("append-policy-id", func(t *testing.T) { + token := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-policy-id=" + thirdPolicy.ID, + "-description=test token", + }) + + require.Len(t, token.Policies, 3) + }) +} + func TestTokenUpdateCommand_JSON(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/website/content/commands/acl/token/update.mdx b/website/content/commands/acl/token/update.mdx index 28158e6db79c..19441e1020b0 100644 --- a/website/content/commands/acl/token/update.mdx +++ b/website/content/commands/acl/token/update.mdx @@ -36,9 +36,15 @@ Usage: `consul acl token update [options]` - `merge-node-identities` - Merge the new node identities with the existing node identities. -- `-merge-policies` - Merge the new policies with the existing policies. +- `-merge-policies` - Deprecated. Merge the new policies with the existing policies. -- `-merge-roles` - Merge the new roles with the existing roles. +~> This is deprecated and will be removed in a future Consul version. Use `append-policy-id` or `append-policy-name` +instead. + +- `-merge-roles` - Deprecated. Merge the new roles with the existing roles. + +~> This is deprecated and will be removed in a future Consul version. Use `append-role-id` or `append-role-name` +instead. - `-merge-service-identities` - Merge the new service identities with the existing service identities. @@ -49,13 +55,25 @@ Usage: `consul acl token update [options]` be specified multiple times. Format is `NODENAME:DATACENTER`. Added in Consul 1.8.1. -- `-policy-id=` - ID of a policy to use for this token. May be specified multiple times. +- `-policy-id=` - ID of a policy to use for this token. Overwrites existing policies. May be specified multiple times. + +- `-policy-name=` - Name of a policy to use for this token. Overwrites existing policies. May be specified multiple times. + +~> `-policy-id` and `-policy-name` will overwrite existing token policies. Use `-append-policy-id` or `-append-policy-name` to retain existing policies. + +- `-append-policy-id=` - ID of policy to be added for this token. The token retains existing policies. May be specified multiple times. + +- `-append-policy-name=` - Name of a policy to be added for this token. The token retains existing policies. May be specified multiple times. + +- `-role-id=` - ID of a role to use for this token. Overwrites existing roles. May be specified multiple times. + +- `-role-name=` - Name of a role to use for this token. Overwrites existing roles. May be specified multiple times. -- `-policy-name=` - Name of a policy to use for this token. May be specified multiple times. +~> `-role-id` and `-role-name` will overwrite existing roles. Use `-append-role-id` or `-append-role-name` to retain the existing roles. -- `-role-id=` - ID of a role to use for this token. May be specified multiple times. +- `-append-role-id=` - ID of a role to add to this token. The token retains existing roles. May be specified multiple times. -- `-role-name=` - Name of a role to use for this token. May be specified multiple times. +- `-append-role-name=` - Name of a role to add to this token. The token retains existing roles. May be specified multiple times. - `-service-identity=` - Name of a service identity to use for this token. May be specified multiple times. Format is the `SERVICENAME` or From 86150c1c8e3dc20db19bd7ef6cab80c91d5a6afe Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 2 Mar 2023 12:11:39 -0600 Subject: [PATCH 074/421] Backport of Suppress AlreadyRegisteredError to fix test retries into release/1.15.x (#16504) * backport of commit a009b044e48159fb42ac9e2231339b7b2b0c5caa * backport of commit 264f9777b00b4c92c219a1cc3196127d9bec2239 --------- Co-authored-by: Chris S. Kim --- lib/telemetry.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/telemetry.go b/lib/telemetry.go index b5815ff64e6c..d07dea96d542 100644 --- a/lib/telemetry.go +++ b/lib/telemetry.go @@ -14,6 +14,7 @@ import ( "github.com/armon/go-metrics/prometheus" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" + prometheuscore "github.com/prometheus/client_golang/prometheus" "github.com/hashicorp/consul/lib/retry" ) @@ -258,7 +259,7 @@ func dogstatdSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, err return sink, nil } -func prometheusSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func prometheusSink(cfg TelemetryConfig, _ string) (metrics.MetricSink, error) { if cfg.PrometheusOpts.Expiration.Nanoseconds() < 1 { return nil, nil @@ -266,12 +267,19 @@ func prometheusSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, e sink, err := prometheus.NewPrometheusSinkFrom(cfg.PrometheusOpts) if err != nil { + // During testing we may try to register the same metrics collector + // multiple times in a single run (e.g. a metrics test fails and + // we attempt a retry), resulting in an AlreadyRegisteredError. + // Suppress this and move on. + if errors.As(err, &prometheuscore.AlreadyRegisteredError{}) { + return nil, nil + } return nil, err } return sink, nil } -func circonusSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func circonusSink(cfg TelemetryConfig, _ string) (metrics.MetricSink, error) { token := cfg.CirconusAPIToken url := cfg.CirconusSubmissionURL if token == "" && url == "" { @@ -337,7 +345,6 @@ func configureSinks(cfg TelemetryConfig, memSink metrics.MetricSink) (metrics.Fa addSink(statsdSink) addSink(dogstatdSink) addSink(circonusSink) - addSink(circonusSink) addSink(prometheusSink) if len(sinks) > 0 { From c3fb1654aaa7ce4f8142dc270c7f86125cd600b7 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 2 Mar 2023 15:17:22 -0600 Subject: [PATCH 075/421] backport of commit 20cd7f1f483d7d0f59416738e81e125bbdc1ace6 (#16510) Co-authored-by: Chris S. Kim --- agent/dns_test.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/agent/dns_test.go b/agent/dns_test.go index 501564c66c2e..1cc74213890b 100644 --- a/agent/dns_test.go +++ b/agent/dns_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/serf/coordinate" "github.com/miekg/dns" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" "github.com/hashicorp/consul/agent/config" "github.com/hashicorp/consul/agent/consul" @@ -4883,21 +4884,26 @@ func TestDNS_TCP_and_UDP_Truncate(t *testing.T) { services := []string{"normal", "truncated"} for index, service := range services { numServices := (index * 5000) + 2 + var eg errgroup.Group for i := 1; i < numServices; i++ { - args := &structs.RegisterRequest{ - Datacenter: "dc1", - Node: fmt.Sprintf("%s-%d.acme.com", service, i), - Address: fmt.Sprintf("127.%d.%d.%d", 0, (i / 255), i%255), - Service: &structs.NodeService{ - Service: service, - Port: 8000, - }, - } + j := i + eg.Go(func() error { + args := &structs.RegisterRequest{ + Datacenter: "dc1", + Node: fmt.Sprintf("%s-%d.acme.com", service, j), + Address: fmt.Sprintf("127.%d.%d.%d", 0, (j / 255), j%255), + Service: &structs.NodeService{ + Service: service, + Port: 8000, + }, + } - var out struct{} - if err := a.RPC(context.Background(), "Catalog.Register", args, &out); err != nil { - t.Fatalf("err: %v", err) - } + var out struct{} + return a.RPC(context.Background(), "Catalog.Register", args, &out) + }) + } + if err := eg.Wait(); err != nil { + t.Fatalf("error registering: %v", err) } // Register an equivalent prepared query. From ee2c7aec4b38611f07528da4149d450e0e0f02d8 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 2 Mar 2023 15:46:28 -0600 Subject: [PATCH 076/421] backport of commit 401052a184d1e7fca61c614f73b1c81bc2736c44 (#16515) Co-authored-by: Michael Hofer Co-authored-by: David Yu --- website/content/docs/architecture/scale.mdx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/website/content/docs/architecture/scale.mdx b/website/content/docs/architecture/scale.mdx index ecf5035f98bc..f05c4b094cfc 100644 --- a/website/content/docs/architecture/scale.mdx +++ b/website/content/docs/architecture/scale.mdx @@ -280,8 +280,4 @@ At scale, using Consul as a backend for Vault results in increased memory and CP In situations where Consul handles large amounts of data and has high write throughput, we recommend adding monitoring for the [capacity and health of raft replication on servers](/consul/docs/agent/telemetry#raft-replication-capacity-issues). If the server experiences heavy load when the size of its stored data is large enough, a follower may be unable to catch up on replication and become a voter after restarting. This situation occurs when the time it takes for a server to restore from disk takes longer than it takes for the leader to write a new snapshot and truncate its logs. Refer to [Raft snapshots](#raft-snapshots) for more information. -<<<<<<< HEAD Vault v1.4 and higher provides [integrated storage](/vault/docs/concepts/integrated-storage) as its recommended storage option. If you currently use Consul as a storage backend for Vault, we recommend switching to integrated storage. For a comparison between Vault's integrated storage and Consul as a backend for Vault, refer to [storage backends in the Vault documentation](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). For detailed guidance on migrating the Vault backend from Consul to Vault's integrated storage, refer to the [storage migration tutorial](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). Integrated storage improves resiliency by preventing a Consul outage from also affecting Vault functionality. -======= -Vault v1.4 and higher provides [integrated storage](/vault/docs/concepts/integrated-storage) as its recommended storage option. If you currently use Consul as a storage backend for Vault, we recommend switching to integrated storage. For a comparison between Vault's integrated storage and Consul as a backend for Vault, refer to [storage backends in the Vault documentation](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). For detailed guidance on migrating the Vault backend from Consul to Vault's integrated storage, refer to the [storage migration tutorial](/vault/docs/configuration/storage#integrated-storage-vs-consul-as-vault-storage). Integrated storage improves resiliency by preventing a Consul outage from also affecting Vault functionality. ->>>>>>> main From f51d12c9520618e13d4d44e81d951ff3b1e1ca4c Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 10:13:25 -0600 Subject: [PATCH 077/421] Backport of Add ServiceResolver RequestTimeout for route timeouts to make TerminatingGateway upstream timeouts configurable into release/1.15.x (#16520) * backport of commit 29768f27aad29fcd8120daf080b06b33c9f22373 * backport of commit 1f26c2a6b4a8f9879cbf4dd6056a94625b753728 * Add RequestTimeout field * Add changelog entry --------- Co-authored-by: Andrew Stucki --- .changelog/16495.txt | 3 + agent/consul/discoverychain/compile.go | 1 + agent/proxycfg/testing_ingress_gateway.go | 6 + agent/proxycfg/testing_mesh_gateway.go | 2 + agent/proxycfg/testing_upstreams.go | 12 + agent/structs/config_entry_discoverychain.go | 21 + agent/structs/discovery_chain.go | 1 + agent/xds/routes.go | 13 +- ...and-failover-to-cluster-peer.latest.golden | 31 +- ...oxy-with-chain-and-overrides.latest.golden | 3 +- ...and-redirect-to-cluster-peer.latest.golden | 31 +- ...roxy-with-chain-external-sni.latest.golden | 31 +- .../connect-proxy-with-chain.latest.golden | 31 +- ...gress-grpc-multiple-services.latest.golden | 46 +- ...gress-http-multiple-services.latest.golden | 6 +- ...-sds-listener-level-wildcard.latest.golden | 46 +- ...ress-with-sds-listener-level.latest.golden | 46 +- ...-sds-service-level-mixed-tls.latest.golden | 54 +- ...gress-with-sds-service-level.latest.golden | 54 +- api/config_entry_discoverychain.go | 1 + proto/pbconfigentry/config_entry.gen.go | 2 + proto/pbconfigentry/config_entry.pb.go | 1996 +++++++++-------- proto/pbconfigentry/config_entry.proto | 2 + .../config-entries/service-resolver.mdx | 6 + 24 files changed, 1270 insertions(+), 1175 deletions(-) create mode 100644 .changelog/16495.txt diff --git a/.changelog/16495.txt b/.changelog/16495.txt new file mode 100644 index 000000000000..4b8ee933ed0e --- /dev/null +++ b/.changelog/16495.txt @@ -0,0 +1,3 @@ +```release-note:improvement +mesh: Add ServiceResolver RequestTimeout for route timeouts to make request timeouts configurable +``` diff --git a/agent/consul/discoverychain/compile.go b/agent/consul/discoverychain/compile.go index 7af98bc06a1d..ead109d419e4 100644 --- a/agent/consul/discoverychain/compile.go +++ b/agent/consul/discoverychain/compile.go @@ -978,6 +978,7 @@ RESOLVE_AGAIN: Default: resolver.IsDefault(), Target: target.ID, ConnectTimeout: connectTimeout, + RequestTimeout: resolver.RequestTimeout, }, LoadBalancer: resolver.LoadBalancer, } diff --git a/agent/proxycfg/testing_ingress_gateway.go b/agent/proxycfg/testing_ingress_gateway.go index f2f5ab67f77d..bca283f18505 100644 --- a/agent/proxycfg/testing_ingress_gateway.go +++ b/agent/proxycfg/testing_ingress_gateway.go @@ -700,11 +700,13 @@ func TestConfigSnapshotIngress_HTTPMultipleServices(t testing.T) *ConfigSnapshot Kind: structs.ServiceResolver, Name: "foo", ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "bar", ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, } @@ -855,11 +857,13 @@ func TestConfigSnapshotIngress_GRPCMultipleServices(t testing.T) *ConfigSnapshot Kind: structs.ServiceResolver, Name: "foo", ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "bar", ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, } @@ -1213,12 +1217,14 @@ func TestConfigSnapshotIngressGatewayWithChain( Name: "web", EnterpriseMeta: *webEntMeta, ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "foo", EnterpriseMeta: *fooEntMeta, ConnectTimeout: 22 * time.Second, + RequestTimeout: 22 * time.Second, }, } diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go index f6b463f9d3fc..f78452f26929 100644 --- a/agent/proxycfg/testing_mesh_gateway.go +++ b/agent/proxycfg/testing_mesh_gateway.go @@ -182,6 +182,7 @@ func TestConfigSnapshotMeshGateway(t testing.T, variant string, nsFn func(ns *st Kind: structs.ServiceResolver, Name: "bar", ConnectTimeout: 10 * time.Second, + RequestTimeout: 10 * time.Second, Subsets: map[string]structs.ServiceResolverSubset{ "v1": { Filter: "Service.Meta.Version == 1", @@ -687,6 +688,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index cb38a3de16ad..3bc09048acf8 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -250,6 +250,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, ) case "external-sni": @@ -263,6 +264,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, ) case "failover": @@ -271,6 +273,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Service: "fail", @@ -293,6 +296,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Datacenters: []string{"dc2"}, @@ -306,6 +310,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Targets: []structs.ServiceResolverFailoverTarget{ @@ -321,6 +326,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Redirect: &structs.ServiceResolverRedirect{ Peer: "cluster-01", }, @@ -341,6 +347,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Datacenters: []string{"dc2", "dc3"}, @@ -363,6 +370,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Datacenters: []string{"dc2"}, @@ -385,6 +393,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { Datacenters: []string{"dc2", "dc3"}, @@ -446,6 +455,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ Kind: structs.ProxyDefaults, @@ -497,6 +507,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ Kind: structs.ProxyDefaults, @@ -528,6 +539,7 @@ func setupTestVariationDiscoveryChain( Kind: structs.ServiceResolver, Name: "db", ConnectTimeout: 33 * time.Second, + RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ Kind: structs.ProxyDefaults, diff --git a/agent/structs/config_entry_discoverychain.go b/agent/structs/config_entry_discoverychain.go index 2444e90101cf..9f80acf682ba 100644 --- a/agent/structs/config_entry_discoverychain.go +++ b/agent/structs/config_entry_discoverychain.go @@ -857,6 +857,11 @@ type ServiceResolverConfigEntry struct { // to this service. ConnectTimeout time.Duration `json:",omitempty" alias:"connect_timeout"` + // RequestTimeout is the timeout for an HTTP request to complete before + // the connection is automatically terminated. If unspecified, defaults + // to 15 seconds. + RequestTimeout time.Duration `json:",omitempty" alias:"request_timeout"` + // LoadBalancer determines the load balancing policy and configuration for services // issuing requests to this upstream service. LoadBalancer *LoadBalancer `json:",omitempty" alias:"load_balancer"` @@ -870,14 +875,19 @@ func (e *ServiceResolverConfigEntry) MarshalJSON() ([]byte, error) { type Alias ServiceResolverConfigEntry exported := &struct { ConnectTimeout string `json:",omitempty"` + RequestTimeout string `json:",omitempty"` *Alias }{ ConnectTimeout: e.ConnectTimeout.String(), + RequestTimeout: e.RequestTimeout.String(), Alias: (*Alias)(e), } if e.ConnectTimeout == 0 { exported.ConnectTimeout = "" } + if e.RequestTimeout == 0 { + exported.RequestTimeout = "" + } return json.Marshal(exported) } @@ -886,6 +896,7 @@ func (e *ServiceResolverConfigEntry) UnmarshalJSON(data []byte) error { type Alias ServiceResolverConfigEntry aux := &struct { ConnectTimeout string + RequestTimeout string *Alias }{ Alias: (*Alias)(e), @@ -899,6 +910,11 @@ func (e *ServiceResolverConfigEntry) UnmarshalJSON(data []byte) error { return err } } + if aux.RequestTimeout != "" { + if e.RequestTimeout, err = time.ParseDuration(aux.RequestTimeout); err != nil { + return err + } + } return nil } @@ -919,6 +935,7 @@ func (e *ServiceResolverConfigEntry) IsDefault() bool { e.Redirect == nil && len(e.Failover) == 0 && e.ConnectTimeout == 0 && + e.RequestTimeout == 0 && e.LoadBalancer == nil } @@ -1117,6 +1134,10 @@ func (e *ServiceResolverConfigEntry) Validate() error { return fmt.Errorf("Bad ConnectTimeout '%s', must be >= 0", e.ConnectTimeout) } + if e.RequestTimeout < 0 { + return fmt.Errorf("Bad RequestTimeout '%s', must be >= 0", e.RequestTimeout) + } + if e.LoadBalancer != nil { lb := e.LoadBalancer diff --git a/agent/structs/discovery_chain.go b/agent/structs/discovery_chain.go index 6de1cd36c467..dc737e7479d6 100644 --- a/agent/structs/discovery_chain.go +++ b/agent/structs/discovery_chain.go @@ -116,6 +116,7 @@ func (s *DiscoveryGraphNode) MapKey() string { type DiscoveryResolver struct { Default bool `json:",omitempty"` ConnectTimeout time.Duration `json:",omitempty"` + RequestTimeout time.Duration `json:",omitempty"` Target string `json:",omitempty"` Failover *DiscoveryFailover `json:",omitempty"` } diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 46d8d2c0a8e1..e80c3c2476e2 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -218,7 +218,7 @@ func (s *ResourceGenerator) makeRoutes( if resolver.LoadBalancer != nil { lb = resolver.LoadBalancer } - route, err := makeNamedDefaultRouteWithLB(clusterName, lb, autoHostRewrite) + route, err := makeNamedDefaultRouteWithLB(clusterName, lb, resolver.RequestTimeout, autoHostRewrite) if err != nil { s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) return nil, err @@ -228,7 +228,7 @@ func (s *ResourceGenerator) makeRoutes( // If there is a service-resolver for this service then also setup routes for each subset for name := range resolver.Subsets { clusterName = connect.ServiceSNI(svc.Name, name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - route, err := makeNamedDefaultRouteWithLB(clusterName, lb, true) + route, err := makeNamedDefaultRouteWithLB(clusterName, lb, resolver.RequestTimeout, true) if err != nil { s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) return nil, err @@ -282,7 +282,7 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho return resources, nil } -func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, autoHostRewrite bool) (*envoy_route_v3.RouteConfiguration, error) { +func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, timeout time.Duration, autoHostRewrite bool) (*envoy_route_v3.RouteConfiguration, error) { action := makeRouteActionFromName(clusterName) if err := injectLBToRouteAction(lb, action.Route); err != nil { @@ -296,6 +296,10 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, a } } + if timeout != 0 { + action.Route.Timeout = durationpb.New(timeout) + } + return &envoy_route_v3.RouteConfiguration{ Name: clusterName, VirtualHosts: []*envoy_route_v3.VirtualHost{ @@ -637,6 +641,9 @@ func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( return nil, fmt.Errorf("failed to apply load balancer configuration to route action: %v", err) } + if startNode.Resolver.RequestTimeout > 0 { + routeAction.Route.Timeout = durationpb.New(startNode.Resolver.RequestTimeout) + } defaultRoute := &envoy_route_v3.Route{ Match: makeDefaultRouteMatch(), Action: routeAction, diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden index 547b923b0d6e..095330c68404 100644 --- a/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -1,30 +1,31 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "db", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ { - "name": "db", - "domains": [ + "name": "db", + "domains": [ "*" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "33s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.latest.golden index f5b65496101a..c40cfe9e90ef 100644 --- a/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.latest.golden +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-overrides.latest.golden @@ -16,7 +16,8 @@ "prefix": "/" }, "route": { - "cluster": "78ebd528~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "cluster": "78ebd528~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "33s" } } ] diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index e63a643ef7d6..cf29d6729a0a 100644 --- a/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -1,30 +1,31 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "db", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ { - "name": "db", - "domains": [ + "name": "db", + "domains": [ "*" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "db.default.cluster-01.external.peer1.domain" + "route": { + "cluster": "db.default.cluster-01.external.peer1.domain", + "timeout": "33s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-external-sni.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-external-sni.latest.golden index 547b923b0d6e..095330c68404 100644 --- a/agent/xds/testdata/routes/connect-proxy-with-chain-external-sni.latest.golden +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-external-sni.latest.golden @@ -1,30 +1,31 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "db", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ { - "name": "db", - "domains": [ + "name": "db", + "domains": [ "*" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "33s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain.latest.golden index 547b923b0d6e..095330c68404 100644 --- a/agent/xds/testdata/routes/connect-proxy-with-chain.latest.golden +++ b/agent/xds/testdata/routes/connect-proxy-with-chain.latest.golden @@ -1,30 +1,31 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "db", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "db", + "virtualHosts": [ { - "name": "db", - "domains": [ + "name": "db", + "domains": [ "*" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "33s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-grpc-multiple-services.latest.golden b/agent/xds/testdata/routes/ingress-grpc-multiple-services.latest.golden index 2822f903d7c0..006bf5157b08 100644 --- a/agent/xds/testdata/routes/ingress-grpc-multiple-services.latest.golden +++ b/agent/xds/testdata/routes/ingress-grpc-multiple-services.latest.golden @@ -1,50 +1,52 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "8080", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "8080", + "virtualHosts": [ { - "name": "foo", - "domains": [ + "name": "foo", + "domains": [ "test1.example.com", "test2.example.com", "test2.example.com:8080", "test1.example.com:8080" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] }, { - "name": "bar", - "domains": [ + "name": "bar", + "domains": [ "bar.ingress.*", "bar.ingress.*:8080" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-http-multiple-services.latest.golden b/agent/xds/testdata/routes/ingress-http-multiple-services.latest.golden index 4e7cfc422dc9..507c66a46d5b 100644 --- a/agent/xds/testdata/routes/ingress-http-multiple-services.latest.golden +++ b/agent/xds/testdata/routes/ingress-http-multiple-services.latest.golden @@ -60,7 +60,8 @@ "prefix": "/" }, "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] @@ -77,7 +78,8 @@ "prefix": "/" }, "route": { - "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.latest.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.latest.golden index 3c251942ec1c..cedfc99f6550 100644 --- a/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.latest.golden +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.latest.golden @@ -1,48 +1,50 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ { - "name": "web", - "domains": [ + "name": "web", + "domains": [ "web.ingress.*", "web.ingress.*:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] }, { - "name": "foo", - "domains": [ + "name": "foo", + "domains": [ "foo.ingress.*", "foo.ingress.*:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level.latest.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level.latest.golden index 0dc02c505695..3f2631217daf 100644 --- a/agent/xds/testdata/routes/ingress-with-sds-listener-level.latest.golden +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level.latest.golden @@ -1,48 +1,50 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ { - "name": "web", - "domains": [ + "name": "web", + "domains": [ "www.example.com", "www.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] }, { - "name": "foo", - "domains": [ + "name": "foo", + "domains": [ "foo.example.com", "foo.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.latest.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.latest.golden index 44ab48e588bf..7539ad4feb1a 100644 --- a/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.latest.golden +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.latest.golden @@ -1,55 +1,57 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ { - "name": "foo", - "domains": [ + "name": "foo", + "domains": [ "foo.example.com", "foo.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true }, { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191_web", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ { - "name": "web", - "domains": [ + "name": "web", + "domains": [ "www.example.com", "www.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level.latest.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level.latest.golden index 1e207d6efca3..6009fac72314 100644 --- a/agent/xds/testdata/routes/ingress-with-sds-service-level.latest.golden +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level.latest.golden @@ -1,55 +1,57 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191_foo", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_foo", + "virtualHosts": [ { - "name": "foo", - "domains": [ + "name": "foo", + "domains": [ "foo.example.com", "foo.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true }, { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "9191_web", - "virtualHosts": [ + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ { - "name": "web", - "domains": [ + "name": "web", + "domains": [ "www.example.com", "www.example.com:9191" ], - "routes": [ + "routes": [ { - "match": { - "prefix": "/" + "match": { + "prefix": "/" }, - "route": { - "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "timeout": "22s" } } ] } ], - "validateClusters": true + "validateClusters": true } ], - "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" } \ No newline at end of file diff --git a/api/config_entry_discoverychain.go b/api/config_entry_discoverychain.go index ff92125dcdc9..acc05e13e1f0 100644 --- a/api/config_entry_discoverychain.go +++ b/api/config_entry_discoverychain.go @@ -167,6 +167,7 @@ type ServiceResolverConfigEntry struct { Redirect *ServiceResolverRedirect `json:",omitempty"` Failover map[string]ServiceResolverFailover `json:",omitempty"` ConnectTimeout time.Duration `json:",omitempty" alias:"connect_timeout"` + RequestTimeout time.Duration `json:",omitempty" alias:"request_timeout"` // LoadBalancer determines the load balancing policy and configuration for services // issuing requests to this upstream service. diff --git a/proto/pbconfigentry/config_entry.gen.go b/proto/pbconfigentry/config_entry.gen.go index d4f2404b1f27..00846d3be7cf 100644 --- a/proto/pbconfigentry/config_entry.gen.go +++ b/proto/pbconfigentry/config_entry.gen.go @@ -1343,6 +1343,7 @@ func ServiceResolverToStructs(s *ServiceResolver, t *structs.ServiceResolverConf } } t.ConnectTimeout = structs.DurationFromProto(s.ConnectTimeout) + t.RequestTimeout = structs.DurationFromProto(s.RequestTimeout) if s.LoadBalancer != nil { var x structs.LoadBalancer LoadBalancerToStructs(s.LoadBalancer, &x) @@ -1385,6 +1386,7 @@ func ServiceResolverFromStructs(t *structs.ServiceResolverConfigEntry, s *Servic } } s.ConnectTimeout = structs.DurationToProto(t.ConnectTimeout) + s.RequestTimeout = structs.DurationToProto(t.RequestTimeout) if t.LoadBalancer != nil { var x LoadBalancer LoadBalancerFromStructs(t.LoadBalancer, &x) diff --git a/proto/pbconfigentry/config_entry.pb.go b/proto/pbconfigentry/config_entry.pb.go index c528cd36feee..442def1c8174 100644 --- a/proto/pbconfigentry/config_entry.pb.go +++ b/proto/pbconfigentry/config_entry.pb.go @@ -1171,6 +1171,8 @@ type ServiceResolver struct { ConnectTimeout *durationpb.Duration `protobuf:"bytes,5,opt,name=ConnectTimeout,proto3" json:"ConnectTimeout,omitempty"` LoadBalancer *LoadBalancer `protobuf:"bytes,6,opt,name=LoadBalancer,proto3" json:"LoadBalancer,omitempty"` Meta map[string]string `protobuf:"bytes,7,rep,name=Meta,proto3" json:"Meta,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // mog: func-to=structs.DurationFromProto func-from=structs.DurationToProto + RequestTimeout *durationpb.Duration `protobuf:"bytes,8,opt,name=RequestTimeout,proto3" json:"RequestTimeout,omitempty"` } func (x *ServiceResolver) Reset() { @@ -1254,6 +1256,13 @@ func (x *ServiceResolver) GetMeta() map[string]string { return nil } +func (x *ServiceResolver) GetRequestTimeout() *durationpb.Duration { + if x != nil { + return x.RequestTimeout + } + return nil +} + // mog annotation: // // target=github.com/hashicorp/consul/agent/structs.ServiceResolverSubset @@ -5459,7 +5468,7 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x67, 0x12, 0x38, 0x0a, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x50, 0x65, 0x65, 0x72, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x4d, - 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x73, 0x22, 0xf6, 0x06, 0x0a, 0x0f, + 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x73, 0x22, 0xb9, 0x07, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x12, 0x24, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, @@ -5496,958 +5505,962 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x78, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x52, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x4d, 0x65, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x1a, 0x78, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x65, + 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x52, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, + 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x7b, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x54, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, + 0x76, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x37, + 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x15, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, + 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x4f, + 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xc9, 0x01, 0x0a, 0x17, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, + 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, + 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x73, 0x12, 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, + 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x22, 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, + 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, + 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x50, 0x65, 0x65, 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x5d, 0x0a, + 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, - 0x73, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7b, - 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x54, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x69, + 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x52, 0x69, + 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, 0x0a, 0x12, + 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, + 0x52, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x22, 0x64, + 0x0a, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, + 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, + 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x61, + 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, + 0x53, 0x69, 0x7a, 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x68, + 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, 0x01, 0x0a, + 0x0a, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x43, 0x6f, + 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x22, 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x03, + 0x54, 0x54, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, + 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0x98, 0x03, + 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, 0x0a, 0x09, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, + 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x1a, + 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8f, 0x02, 0x0a, 0x14, 0x49, 0x6e, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, + 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, + 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, + 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x18, 0x0a, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, + 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, + 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, + 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, + 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x51, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x03, 0x54, + 0x4c, 0x53, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xb7, 0x06, 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x48, 0x6f, + 0x73, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x37, 0x0a, 0x09, 0x4d, - 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x15, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, - 0x73, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, - 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xc9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, 0x64, 0x69, 0x72, - 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, - 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, - 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, - 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, - 0x65, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x12, - 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, - 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, - 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, - 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, - 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, - 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, - 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, - 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, - 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, - 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x5d, 0x0a, 0x0e, 0x52, 0x69, - 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x69, 0x6e, 0x67, 0x48, - 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, - 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, 0x0a, 0x12, 0x4c, 0x65, 0x61, - 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x65, - 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0c, 0x48, - 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0e, 0x52, - 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, - 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, - 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, - 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, - 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, - 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x43, 0x68, - 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x0a, 0x48, 0x61, - 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1e, - 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x57, - 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, 0x6f, - 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x22, - 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x03, 0x54, 0x54, 0x4c, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0x98, 0x03, 0x0a, 0x0e, 0x49, - 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x49, 0x0a, - 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x53, - 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0f, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, + 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, + 0x4d, 0x65, 0x74, 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, + 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, - 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x37, 0x0a, 0x09, - 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8f, 0x02, 0x0a, 0x14, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x26, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, + 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, + 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, - 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, + 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6b, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, - 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, - 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, - 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, - 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x51, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, - 0x54, 0x4c, 0x53, 0x22, 0xb7, 0x06, 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x48, 0x6f, - 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, - 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, - 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, - 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0e, 0x4d, - 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, - 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, - 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, - 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, - 0x17, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, - 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x55, - 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, - 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x53, - 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, - 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x07, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x56, 0x0a, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, 0x06, - 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, - 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x12, - 0x4e, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x66, 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x18, - 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x67, 0x0a, 0x17, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, 0x0a, 0x03, 0x53, + 0x44, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, 0x13, 0x48, 0x54, + 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, + 0x73, 0x12, 0x55, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4c, 0x65, - 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x4c, - 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, - 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x4c, 0x65, 0x67, 0x61, 0x63, - 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4e, - 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, + 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, + 0x73, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x36, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, + 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, - 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, - 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x48, 0x54, - 0x54, 0x50, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, - 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x0a, - 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1c, 0x0a, 0x09, - 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x5c, 0x0a, 0x06, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, - 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x68, - 0x6f, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, - 0x64, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x72, 0x65, 0x66, - 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x12, 0x16, 0x0a, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x65, 0x67, 0x65, - 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x16, - 0x0a, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x22, 0xb6, 0x08, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x44, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x69, 0x0a, 0x10, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x5a, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x12, 0x4b, 0x0a, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, - 0x12, 0x20, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x49, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, - 0x4e, 0x49, 0x12, 0x64, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5a, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74, - 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, + 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x4d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, - 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x3c, 0x0a, 0x19, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x42, 0x61, 0x6c, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0d, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x5a, 0x0a, 0x0f, 0x45, 0x6e, - 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, - 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, - 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x74, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, - 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6c, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, - 0x65, 0x63, 0x74, 0x6c, 0x79, 0x22, 0x5f, 0x0a, 0x11, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x04, 0x4d, 0x6f, - 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0xa6, 0x06, 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, - 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x6f, 0x0a, 0x0c, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x12, 0x47, - 0x0a, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, + 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, 0x65, 0x72, 0x6d, + 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, - 0x52, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb0, 0x01, 0x0a, 0x0a, 0x45, 0x78, 0x70, 0x6f, - 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x4c, 0x69, - 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, - 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, - 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x50, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, - 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x12, 0x28, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, - 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xbf, 0x01, 0x0a, 0x15, 0x55, - 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x09, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, + 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, 0x72, 0x65, 0x63, + 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, + 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, + 0x49, 0x44, 0x12, 0x4e, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, + 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x10, + 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, + 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, + 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, + 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, + 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0c, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x4c, 0x65, + 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, 0x13, 0x49, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x52, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x04, 0x48, 0x54, 0x54, 0x50, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, + 0x1e, 0x0a, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, + 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x5c, 0x0a, + 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, + 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, + 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x52, + 0x65, 0x67, 0x65, 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x65, 0x67, 0x65, + 0x78, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x22, 0xb6, 0x08, 0x0a, 0x0f, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1a, 0x0a, + 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x44, 0x0a, 0x04, 0x4d, 0x6f, 0x64, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x08, 0x44, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x69, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x5a, 0x0a, 0x0b, 0x4d, 0x65, + 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4b, 0x0a, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, + 0x78, 0x70, 0x6f, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x45, 0x78, 0x70, + 0x6f, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, + 0x4e, 0x49, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x53, 0x4e, 0x49, 0x12, 0x64, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x55, 0x70, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5a, 0x0a, 0x0b, 0x44, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x49, 0x6e, 0x62, 0x6f, 0x75, + 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x34, 0x0a, + 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, + 0x74, 0x4d, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x3c, 0x0a, 0x19, 0x42, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x42, 0x61, + 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, + 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2e, 0x4d, 0x65, + 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x5a, 0x0a, + 0x0f, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, + 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x74, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, + 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x4f, 0x75, 0x74, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, + 0x12, 0x26, 0x0a, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x64, + 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6c, 0x79, 0x22, 0x5f, 0x0a, 0x11, 0x4d, 0x65, 0x73, 0x68, + 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, + 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x8a, 0x05, 0x0a, - 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, - 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, - 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, - 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x2c, 0x0a, - 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, - 0x4f, 0x4e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x2a, 0x0a, 0x10, 0x45, - 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x12, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, - 0x4d, 0x0a, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x69, - 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, + 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, + 0x6f, 0x64, 0x65, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x6f, 0x0a, 0x0c, 0x45, 0x78, 0x70, + 0x6f, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x73, 0x12, 0x47, 0x0a, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x50, + 0x61, 0x74, 0x68, 0x52, 0x05, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb0, 0x01, 0x0a, 0x0a, 0x45, + 0x78, 0x70, 0x6f, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, + 0x68, 0x12, 0x24, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x50, 0x6f, + 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x73, 0x65, 0x64, 0x46, 0x72, 0x6f, + 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x50, 0x61, + 0x72, 0x73, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0xbf, 0x01, + 0x0a, 0x15, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x09, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x5a, 0x0a, 0x0b, 0x4d, 0x65, 0x73, - 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, + 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x09, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x08, + 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x3e, 0x0a, 0x1a, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x42, 0x61, 0x6c, 0x61, 0x6e, - 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0x9e, 0x01, 0x0a, 0x0e, 0x55, 0x70, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, - 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x50, - 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x12, 0x35, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x4d, 0x61, 0x78, 0x46, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x4d, - 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x17, 0x45, 0x6e, - 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x76, 0x65, 0x35, 0x78, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x17, 0x45, 0x6e, 0x66, - 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, - 0x65, 0x35, 0x78, 0x78, 0x22, 0x45, 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x41, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xb6, 0x02, 0x0a, 0x0a, - 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4f, 0x0a, 0x04, 0x4d, 0x65, - 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x57, 0x0a, 0x09, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, + 0x8a, 0x05, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, + 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, - 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x50, - 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, + 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, + 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, + 0x12, 0x2c, 0x0a, 0x11, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x45, 0x6e, 0x76, + 0x6f, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x2a, + 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x53, + 0x4f, 0x4e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x4d, 0x73, 0x12, 0x4d, 0x0a, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, 0x6e, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x8b, 0x02, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, - 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, - 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x54, 0x0a, 0x08, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x12, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x12, 0x4c, 0x61, 0x73, 0x74, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x8c, - 0x02, 0x0a, 0x12, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, - 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, - 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x5d, 0x0a, 0x08, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x41, 0x2e, 0x68, 0x61, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x06, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x73, 0x12, 0x69, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, + 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x5a, 0x0a, 0x0b, + 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x4d, 0x65, 0x73, + 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x3e, 0x0a, 0x1a, 0x42, 0x61, 0x6c, 0x61, + 0x6e, 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x42, 0x61, + 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0x9e, 0x01, 0x0a, + 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, + 0x26, 0x0a, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, + 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x12, 0x4d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, + 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x22, 0xa7, 0x01, + 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x69, 0x76, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x12, 0x35, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x4d, + 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x0b, 0x4d, 0x61, 0x78, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x38, 0x0a, + 0x17, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x17, + 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x22, 0x45, 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1c, 0x0a, 0x09, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x09, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, + 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xb6, + 0x02, 0x0a, 0x0a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4f, 0x0a, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, - 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x53, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, + 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x57, + 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, + 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x50, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, - 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xde, 0x01, - 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0c, - 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x54, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x12, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x12, 0x4c, + 0x61, 0x73, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x22, 0x8c, 0x02, 0x0a, 0x12, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x5d, 0x0a, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x41, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x53, 0x0a, 0x03, 0x54, + 0x4c, 0x53, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x4c, 0x53, + 0x22, 0xde, 0x01, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, + 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x5c, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, + 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, + 0x0a, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, + 0x0a, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, + 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, + 0x73, 0x22, 0xb7, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfe, 0x01, 0x0a, 0x0f, + 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, + 0x54, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x5c, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, + 0x17, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x5c, 0x0a, 0x0c, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x43, 0x65, - 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x69, - 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, - 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, - 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0xb7, - 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, - 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, - 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, - 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, - 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfe, 0x01, 0x0a, 0x0f, 0x42, 0x6f, 0x75, - 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x54, 0x0a, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x06, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, - 0x74, 0x61, 0x12, 0x5c, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x42, 0x6f, - 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, - 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, 0x17, 0x42, 0x6f, - 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x5c, 0x0a, 0x0c, 0x43, 0x65, 0x72, - 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0xe6, 0x01, 0x0a, + 0x11, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x12, 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x50, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x50, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x37, 0x0a, 0x09, + 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x99, 0x03, 0x0a, 0x09, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, + 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, + 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, + 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x52, 0x75, + 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xf9, 0x01, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, + 0x75, 0x6c, 0x65, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, + 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, + 0x73, 0x12, 0x4a, 0x0a, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x4e, 0x0a, + 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x69, - 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x22, 0xc4, 0x02, + 0x0a, 0x09, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x07, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4e, 0x0a, + 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x48, 0x0a, + 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x4b, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, + 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0xe6, 0x01, 0x0a, 0x11, 0x49, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, - 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, + 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x75, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4e, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, + 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x0e, + 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4f, + 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, - 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x65, - 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x69, - 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, - 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, - 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x99, 0x03, 0x0a, 0x09, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x12, 0x4e, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, - 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, 0x50, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, + 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb3, 0x01, 0x0a, 0x0b, 0x48, 0x54, + 0x54, 0x50, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x07, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x0a, + 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, + 0x69, 0x74, 0x65, 0x52, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, + 0x20, 0x0a, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, + 0x68, 0x22, 0xc2, 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x52, 0x75, 0x6c, 0x65, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x45, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x12, 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf9, - 0x01, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x75, 0x6c, 0x65, - 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4a, - 0x0a, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x52, 0x07, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x4e, 0x0a, 0x08, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x22, 0xc4, 0x02, 0x0a, 0x09, 0x48, - 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4e, 0x0a, 0x06, 0x4d, 0x65, - 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, - 0x6f, 0x64, 0x52, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x48, 0x0a, 0x04, 0x50, 0x61, - 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x04, - 0x50, 0x61, 0x74, 0x68, 0x12, 0x4b, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, + 0x0a, 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, + 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x12, 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x51, 0x75, 0x65, 0x72, - 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, - 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x75, 0x0a, 0x0d, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x12, 0x4e, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, - 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x0e, 0x48, 0x54, 0x54, - 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4f, 0x0a, 0x05, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb3, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfc, 0x02, 0x0a, 0x08, 0x54, + 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, + 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x52, 0x07, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x0a, 0x55, 0x52, 0x4c, - 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, - 0x52, 0x0a, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, 0x20, 0x0a, 0x0a, - 0x55, 0x52, 0x4c, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0xc2, - 0x02, 0x0a, 0x10, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x12, - 0x52, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, - 0x53, 0x65, 0x74, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, 0x53, - 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0xe1, 0x01, 0x0a, 0x0b, 0x48, 0x54, 0x54, 0x50, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, - 0x4c, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x32, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x46, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, - 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, - 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, - 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x22, 0xfc, 0x02, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x52, - 0x6f, 0x75, 0x74, 0x65, 0x12, 0x4d, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x52, 0x6f, - 0x75, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, - 0x65, 0x74, 0x61, 0x12, 0x52, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, - 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, + 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, - 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x43, 0x50, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, - 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, - 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, - 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, - 0x74, 0x61, 0x2a, 0xfd, 0x01, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x4b, - 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, - 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x01, - 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, 0x6e, - 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, - 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, - 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, - 0x6c, 0x74, 0x73, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x6c, - 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0x06, - 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, 0x6f, 0x75, 0x6e, - 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, 0x12, 0x11, 0x0a, - 0x0d, 0x4b, 0x69, 0x6e, 0x64, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x10, 0x09, - 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x10, 0x0a, 0x2a, 0x26, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, - 0x09, 0x0a, 0x05, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x2a, 0x50, 0x0a, - 0x09, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, - 0x12, 0x18, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x10, 0x02, 0x2a, - 0x7b, 0x0a, 0x0f, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x17, - 0x0a, 0x13, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, - 0x65, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, 0x73, 0x68, 0x47, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x10, - 0x02, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, 0x4f, 0x0a, 0x1a, - 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x69, - 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x48, 0x54, - 0x54, 0x50, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, 0x2a, 0x92, 0x02, - 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, - 0x64, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, - 0x74, 0x68, 0x6f, 0x64, 0x41, 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, - 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, - 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, - 0x68, 0x6f, 0x64, 0x47, 0x65, 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, 0x61, 0x64, 0x10, - 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, - 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, 0x12, 0x18, 0x0a, - 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, - 0x50, 0x61, 0x74, 0x63, 0x68, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, 0x74, 0x10, 0x07, - 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, - 0x68, 0x6f, 0x64, 0x50, 0x75, 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, 0x61, 0x63, 0x65, - 0x10, 0x09, 0x2a, 0xa7, 0x01, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, - 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, - 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, - 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x48, - 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, - 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, - 0x03, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, 0x68, 0x0a, 0x11, - 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, - 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x10, 0x01, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, - 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, - 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, - 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x01, - 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, 0x43, 0xaa, - 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xe2, - 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, - 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x43, 0x50, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x2a, 0xfd, 0x01, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, + 0x0a, 0x0b, 0x4b, 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, + 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, + 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x12, + 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, + 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x42, + 0x6f, 0x75, 0x6e, 0x64, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, 0x08, + 0x12, 0x11, 0x0a, 0x0d, 0x4b, 0x69, 0x6e, 0x64, 0x48, 0x54, 0x54, 0x50, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x10, 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x69, 0x6e, 0x64, 0x54, 0x43, 0x50, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x10, 0x0a, 0x2a, 0x26, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, + 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, + 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, + 0x2a, 0x50, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, + 0x10, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x10, 0x01, 0x12, 0x13, 0x0a, + 0x0f, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x10, 0x02, 0x2a, 0x7b, 0x0a, 0x0f, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, + 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x4d, 0x6f, 0x64, 0x65, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x65, + 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x65, 0x73, 0x68, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x10, 0x03, 0x2a, + 0x4f, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, 0x0a, + 0x14, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x48, 0x54, 0x54, 0x50, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x54, 0x43, 0x50, 0x10, 0x01, + 0x2a, 0x92, 0x02, 0x0a, 0x0f, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x41, 0x6c, 0x6c, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, + 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x47, 0x65, 0x74, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x48, + 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x48, 0x65, + 0x61, 0x64, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x05, + 0x12, 0x18, 0x0a, 0x14, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x50, 0x61, 0x74, 0x63, 0x68, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x48, 0x54, + 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x6f, 0x73, + 0x74, 0x10, 0x07, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x50, 0x75, 0x74, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x48, + 0x54, 0x54, 0x50, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x54, 0x72, + 0x61, 0x63, 0x65, 0x10, 0x09, 0x2a, 0xa7, 0x01, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, + 0x14, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x24, + 0x0a, 0x20, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x10, 0x04, 0x2a, + 0x68, 0x0a, 0x11, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, + 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x10, 0x01, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x61, 0x74, + 0x68, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, + 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x2a, 0x6d, 0x0a, 0x12, 0x48, 0x54, 0x54, + 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x17, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x48, 0x54, 0x54, 0x50, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x74, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x54, 0x54, 0x50, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x45, 0x78, 0x70, 0x72, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, + 0x49, 0x43, 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -6587,105 +6600,106 @@ var file_proto_pbconfigentry_config_entry_proto_depIdxs = []int32{ 91, // 23: hashicorp.consul.internal.configentry.ServiceResolver.ConnectTimeout:type_name -> google.protobuf.Duration 22, // 24: hashicorp.consul.internal.configentry.ServiceResolver.LoadBalancer:type_name -> hashicorp.consul.internal.configentry.LoadBalancer 74, // 25: hashicorp.consul.internal.configentry.ServiceResolver.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceResolver.MetaEntry - 21, // 26: hashicorp.consul.internal.configentry.ServiceResolverFailover.Targets:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailoverTarget - 23, // 27: hashicorp.consul.internal.configentry.LoadBalancer.RingHashConfig:type_name -> hashicorp.consul.internal.configentry.RingHashConfig - 24, // 28: hashicorp.consul.internal.configentry.LoadBalancer.LeastRequestConfig:type_name -> hashicorp.consul.internal.configentry.LeastRequestConfig - 25, // 29: hashicorp.consul.internal.configentry.LoadBalancer.HashPolicies:type_name -> hashicorp.consul.internal.configentry.HashPolicy - 26, // 30: hashicorp.consul.internal.configentry.HashPolicy.CookieConfig:type_name -> hashicorp.consul.internal.configentry.CookieConfig - 91, // 31: hashicorp.consul.internal.configentry.CookieConfig.TTL:type_name -> google.protobuf.Duration - 29, // 32: hashicorp.consul.internal.configentry.IngressGateway.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig - 31, // 33: hashicorp.consul.internal.configentry.IngressGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.IngressListener - 75, // 34: hashicorp.consul.internal.configentry.IngressGateway.Meta:type_name -> hashicorp.consul.internal.configentry.IngressGateway.MetaEntry - 28, // 35: hashicorp.consul.internal.configentry.IngressGateway.Defaults:type_name -> hashicorp.consul.internal.configentry.IngressServiceConfig - 48, // 36: hashicorp.consul.internal.configentry.IngressServiceConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 30, // 37: hashicorp.consul.internal.configentry.GatewayTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig - 32, // 38: hashicorp.consul.internal.configentry.IngressListener.Services:type_name -> hashicorp.consul.internal.configentry.IngressService - 29, // 39: hashicorp.consul.internal.configentry.IngressListener.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig - 33, // 40: hashicorp.consul.internal.configentry.IngressService.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayServiceTLSConfig - 34, // 41: hashicorp.consul.internal.configentry.IngressService.RequestHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers - 34, // 42: hashicorp.consul.internal.configentry.IngressService.ResponseHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers - 76, // 43: hashicorp.consul.internal.configentry.IngressService.Meta:type_name -> hashicorp.consul.internal.configentry.IngressService.MetaEntry - 89, // 44: hashicorp.consul.internal.configentry.IngressService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 48, // 45: hashicorp.consul.internal.configentry.IngressService.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 30, // 46: hashicorp.consul.internal.configentry.GatewayServiceTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig - 77, // 47: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.AddEntry - 78, // 48: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.SetEntry - 36, // 49: hashicorp.consul.internal.configentry.ServiceIntentions.Sources:type_name -> hashicorp.consul.internal.configentry.SourceIntention - 79, // 50: hashicorp.consul.internal.configentry.ServiceIntentions.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceIntentions.MetaEntry - 1, // 51: hashicorp.consul.internal.configentry.SourceIntention.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction - 37, // 52: hashicorp.consul.internal.configentry.SourceIntention.Permissions:type_name -> hashicorp.consul.internal.configentry.IntentionPermission - 2, // 53: hashicorp.consul.internal.configentry.SourceIntention.Type:type_name -> hashicorp.consul.internal.configentry.IntentionSourceType - 80, // 54: hashicorp.consul.internal.configentry.SourceIntention.LegacyMeta:type_name -> hashicorp.consul.internal.configentry.SourceIntention.LegacyMetaEntry - 92, // 55: hashicorp.consul.internal.configentry.SourceIntention.LegacyCreateTime:type_name -> google.protobuf.Timestamp - 92, // 56: hashicorp.consul.internal.configentry.SourceIntention.LegacyUpdateTime:type_name -> google.protobuf.Timestamp - 89, // 57: hashicorp.consul.internal.configentry.SourceIntention.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 1, // 58: hashicorp.consul.internal.configentry.IntentionPermission.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction - 38, // 59: hashicorp.consul.internal.configentry.IntentionPermission.HTTP:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPPermission - 39, // 60: hashicorp.consul.internal.configentry.IntentionHTTPPermission.Header:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPHeaderPermission - 3, // 61: hashicorp.consul.internal.configentry.ServiceDefaults.Mode:type_name -> hashicorp.consul.internal.configentry.ProxyMode - 41, // 62: hashicorp.consul.internal.configentry.ServiceDefaults.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyConfig - 42, // 63: hashicorp.consul.internal.configentry.ServiceDefaults.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig - 43, // 64: hashicorp.consul.internal.configentry.ServiceDefaults.Expose:type_name -> hashicorp.consul.internal.configentry.ExposeConfig - 45, // 65: hashicorp.consul.internal.configentry.ServiceDefaults.UpstreamConfig:type_name -> hashicorp.consul.internal.configentry.UpstreamConfiguration - 49, // 66: hashicorp.consul.internal.configentry.ServiceDefaults.Destination:type_name -> hashicorp.consul.internal.configentry.DestinationConfig - 81, // 67: hashicorp.consul.internal.configentry.ServiceDefaults.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceDefaults.MetaEntry - 93, // 68: hashicorp.consul.internal.configentry.ServiceDefaults.EnvoyExtensions:type_name -> hashicorp.consul.internal.common.EnvoyExtension - 4, // 69: hashicorp.consul.internal.configentry.MeshGatewayConfig.Mode:type_name -> hashicorp.consul.internal.configentry.MeshGatewayMode - 44, // 70: hashicorp.consul.internal.configentry.ExposeConfig.Paths:type_name -> hashicorp.consul.internal.configentry.ExposePath - 46, // 71: hashicorp.consul.internal.configentry.UpstreamConfiguration.Overrides:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig - 46, // 72: hashicorp.consul.internal.configentry.UpstreamConfiguration.Defaults:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig - 89, // 73: hashicorp.consul.internal.configentry.UpstreamConfig.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 47, // 74: hashicorp.consul.internal.configentry.UpstreamConfig.Limits:type_name -> hashicorp.consul.internal.configentry.UpstreamLimits - 48, // 75: hashicorp.consul.internal.configentry.UpstreamConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck - 42, // 76: hashicorp.consul.internal.configentry.UpstreamConfig.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig - 91, // 77: hashicorp.consul.internal.configentry.PassiveHealthCheck.Interval:type_name -> google.protobuf.Duration - 82, // 78: hashicorp.consul.internal.configentry.APIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.APIGateway.MetaEntry - 53, // 79: hashicorp.consul.internal.configentry.APIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.APIGatewayListener - 51, // 80: hashicorp.consul.internal.configentry.APIGateway.Status:type_name -> hashicorp.consul.internal.configentry.Status - 52, // 81: hashicorp.consul.internal.configentry.Status.Conditions:type_name -> hashicorp.consul.internal.configentry.Condition - 55, // 82: hashicorp.consul.internal.configentry.Condition.Resource:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 92, // 83: hashicorp.consul.internal.configentry.Condition.LastTransitionTime:type_name -> google.protobuf.Timestamp - 5, // 84: hashicorp.consul.internal.configentry.APIGatewayListener.Protocol:type_name -> hashicorp.consul.internal.configentry.APIGatewayListenerProtocol - 54, // 85: hashicorp.consul.internal.configentry.APIGatewayListener.TLS:type_name -> hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration - 55, // 86: hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 89, // 87: hashicorp.consul.internal.configentry.ResourceReference.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 83, // 88: hashicorp.consul.internal.configentry.BoundAPIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.BoundAPIGateway.MetaEntry - 57, // 89: hashicorp.consul.internal.configentry.BoundAPIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.BoundAPIGatewayListener - 55, // 90: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 55, // 91: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Routes:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 84, // 92: hashicorp.consul.internal.configentry.InlineCertificate.Meta:type_name -> hashicorp.consul.internal.configentry.InlineCertificate.MetaEntry - 85, // 93: hashicorp.consul.internal.configentry.HTTPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.HTTPRoute.MetaEntry - 55, // 94: hashicorp.consul.internal.configentry.HTTPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 60, // 95: hashicorp.consul.internal.configentry.HTTPRoute.Rules:type_name -> hashicorp.consul.internal.configentry.HTTPRouteRule - 51, // 96: hashicorp.consul.internal.configentry.HTTPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status - 65, // 97: hashicorp.consul.internal.configentry.HTTPRouteRule.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters - 61, // 98: hashicorp.consul.internal.configentry.HTTPRouteRule.Matches:type_name -> hashicorp.consul.internal.configentry.HTTPMatch - 68, // 99: hashicorp.consul.internal.configentry.HTTPRouteRule.Services:type_name -> hashicorp.consul.internal.configentry.HTTPService - 62, // 100: hashicorp.consul.internal.configentry.HTTPMatch.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatch - 6, // 101: hashicorp.consul.internal.configentry.HTTPMatch.Method:type_name -> hashicorp.consul.internal.configentry.HTTPMatchMethod - 63, // 102: hashicorp.consul.internal.configentry.HTTPMatch.Path:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatch - 64, // 103: hashicorp.consul.internal.configentry.HTTPMatch.Query:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatch - 7, // 104: hashicorp.consul.internal.configentry.HTTPHeaderMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatchType - 8, // 105: hashicorp.consul.internal.configentry.HTTPPathMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatchType - 9, // 106: hashicorp.consul.internal.configentry.HTTPQueryMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatchType - 67, // 107: hashicorp.consul.internal.configentry.HTTPFilters.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter - 66, // 108: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrite:type_name -> hashicorp.consul.internal.configentry.URLRewrite - 86, // 109: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.AddEntry - 87, // 110: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.SetEntry - 65, // 111: hashicorp.consul.internal.configentry.HTTPService.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters - 89, // 112: hashicorp.consul.internal.configentry.HTTPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 88, // 113: hashicorp.consul.internal.configentry.TCPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.TCPRoute.MetaEntry - 55, // 114: hashicorp.consul.internal.configentry.TCPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference - 70, // 115: hashicorp.consul.internal.configentry.TCPRoute.Services:type_name -> hashicorp.consul.internal.configentry.TCPService - 51, // 116: hashicorp.consul.internal.configentry.TCPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status - 89, // 117: hashicorp.consul.internal.configentry.TCPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta - 18, // 118: hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverSubset - 20, // 119: hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailover - 120, // [120:120] is the sub-list for method output_type - 120, // [120:120] is the sub-list for method input_type - 120, // [120:120] is the sub-list for extension type_name - 120, // [120:120] is the sub-list for extension extendee - 0, // [0:120] is the sub-list for field type_name + 91, // 26: hashicorp.consul.internal.configentry.ServiceResolver.RequestTimeout:type_name -> google.protobuf.Duration + 21, // 27: hashicorp.consul.internal.configentry.ServiceResolverFailover.Targets:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailoverTarget + 23, // 28: hashicorp.consul.internal.configentry.LoadBalancer.RingHashConfig:type_name -> hashicorp.consul.internal.configentry.RingHashConfig + 24, // 29: hashicorp.consul.internal.configentry.LoadBalancer.LeastRequestConfig:type_name -> hashicorp.consul.internal.configentry.LeastRequestConfig + 25, // 30: hashicorp.consul.internal.configentry.LoadBalancer.HashPolicies:type_name -> hashicorp.consul.internal.configentry.HashPolicy + 26, // 31: hashicorp.consul.internal.configentry.HashPolicy.CookieConfig:type_name -> hashicorp.consul.internal.configentry.CookieConfig + 91, // 32: hashicorp.consul.internal.configentry.CookieConfig.TTL:type_name -> google.protobuf.Duration + 29, // 33: hashicorp.consul.internal.configentry.IngressGateway.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig + 31, // 34: hashicorp.consul.internal.configentry.IngressGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.IngressListener + 75, // 35: hashicorp.consul.internal.configentry.IngressGateway.Meta:type_name -> hashicorp.consul.internal.configentry.IngressGateway.MetaEntry + 28, // 36: hashicorp.consul.internal.configentry.IngressGateway.Defaults:type_name -> hashicorp.consul.internal.configentry.IngressServiceConfig + 48, // 37: hashicorp.consul.internal.configentry.IngressServiceConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 30, // 38: hashicorp.consul.internal.configentry.GatewayTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig + 32, // 39: hashicorp.consul.internal.configentry.IngressListener.Services:type_name -> hashicorp.consul.internal.configentry.IngressService + 29, // 40: hashicorp.consul.internal.configentry.IngressListener.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSConfig + 33, // 41: hashicorp.consul.internal.configentry.IngressService.TLS:type_name -> hashicorp.consul.internal.configentry.GatewayServiceTLSConfig + 34, // 42: hashicorp.consul.internal.configentry.IngressService.RequestHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers + 34, // 43: hashicorp.consul.internal.configentry.IngressService.ResponseHeaders:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers + 76, // 44: hashicorp.consul.internal.configentry.IngressService.Meta:type_name -> hashicorp.consul.internal.configentry.IngressService.MetaEntry + 89, // 45: hashicorp.consul.internal.configentry.IngressService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 48, // 46: hashicorp.consul.internal.configentry.IngressService.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 30, // 47: hashicorp.consul.internal.configentry.GatewayServiceTLSConfig.SDS:type_name -> hashicorp.consul.internal.configentry.GatewayTLSSDSConfig + 77, // 48: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.AddEntry + 78, // 49: hashicorp.consul.internal.configentry.HTTPHeaderModifiers.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderModifiers.SetEntry + 36, // 50: hashicorp.consul.internal.configentry.ServiceIntentions.Sources:type_name -> hashicorp.consul.internal.configentry.SourceIntention + 79, // 51: hashicorp.consul.internal.configentry.ServiceIntentions.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceIntentions.MetaEntry + 1, // 52: hashicorp.consul.internal.configentry.SourceIntention.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction + 37, // 53: hashicorp.consul.internal.configentry.SourceIntention.Permissions:type_name -> hashicorp.consul.internal.configentry.IntentionPermission + 2, // 54: hashicorp.consul.internal.configentry.SourceIntention.Type:type_name -> hashicorp.consul.internal.configentry.IntentionSourceType + 80, // 55: hashicorp.consul.internal.configentry.SourceIntention.LegacyMeta:type_name -> hashicorp.consul.internal.configentry.SourceIntention.LegacyMetaEntry + 92, // 56: hashicorp.consul.internal.configentry.SourceIntention.LegacyCreateTime:type_name -> google.protobuf.Timestamp + 92, // 57: hashicorp.consul.internal.configentry.SourceIntention.LegacyUpdateTime:type_name -> google.protobuf.Timestamp + 89, // 58: hashicorp.consul.internal.configentry.SourceIntention.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 1, // 59: hashicorp.consul.internal.configentry.IntentionPermission.Action:type_name -> hashicorp.consul.internal.configentry.IntentionAction + 38, // 60: hashicorp.consul.internal.configentry.IntentionPermission.HTTP:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPPermission + 39, // 61: hashicorp.consul.internal.configentry.IntentionHTTPPermission.Header:type_name -> hashicorp.consul.internal.configentry.IntentionHTTPHeaderPermission + 3, // 62: hashicorp.consul.internal.configentry.ServiceDefaults.Mode:type_name -> hashicorp.consul.internal.configentry.ProxyMode + 41, // 63: hashicorp.consul.internal.configentry.ServiceDefaults.TransparentProxy:type_name -> hashicorp.consul.internal.configentry.TransparentProxyConfig + 42, // 64: hashicorp.consul.internal.configentry.ServiceDefaults.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig + 43, // 65: hashicorp.consul.internal.configentry.ServiceDefaults.Expose:type_name -> hashicorp.consul.internal.configentry.ExposeConfig + 45, // 66: hashicorp.consul.internal.configentry.ServiceDefaults.UpstreamConfig:type_name -> hashicorp.consul.internal.configentry.UpstreamConfiguration + 49, // 67: hashicorp.consul.internal.configentry.ServiceDefaults.Destination:type_name -> hashicorp.consul.internal.configentry.DestinationConfig + 81, // 68: hashicorp.consul.internal.configentry.ServiceDefaults.Meta:type_name -> hashicorp.consul.internal.configentry.ServiceDefaults.MetaEntry + 93, // 69: hashicorp.consul.internal.configentry.ServiceDefaults.EnvoyExtensions:type_name -> hashicorp.consul.internal.common.EnvoyExtension + 4, // 70: hashicorp.consul.internal.configentry.MeshGatewayConfig.Mode:type_name -> hashicorp.consul.internal.configentry.MeshGatewayMode + 44, // 71: hashicorp.consul.internal.configentry.ExposeConfig.Paths:type_name -> hashicorp.consul.internal.configentry.ExposePath + 46, // 72: hashicorp.consul.internal.configentry.UpstreamConfiguration.Overrides:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig + 46, // 73: hashicorp.consul.internal.configentry.UpstreamConfiguration.Defaults:type_name -> hashicorp.consul.internal.configentry.UpstreamConfig + 89, // 74: hashicorp.consul.internal.configentry.UpstreamConfig.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 47, // 75: hashicorp.consul.internal.configentry.UpstreamConfig.Limits:type_name -> hashicorp.consul.internal.configentry.UpstreamLimits + 48, // 76: hashicorp.consul.internal.configentry.UpstreamConfig.PassiveHealthCheck:type_name -> hashicorp.consul.internal.configentry.PassiveHealthCheck + 42, // 77: hashicorp.consul.internal.configentry.UpstreamConfig.MeshGateway:type_name -> hashicorp.consul.internal.configentry.MeshGatewayConfig + 91, // 78: hashicorp.consul.internal.configentry.PassiveHealthCheck.Interval:type_name -> google.protobuf.Duration + 82, // 79: hashicorp.consul.internal.configentry.APIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.APIGateway.MetaEntry + 53, // 80: hashicorp.consul.internal.configentry.APIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.APIGatewayListener + 51, // 81: hashicorp.consul.internal.configentry.APIGateway.Status:type_name -> hashicorp.consul.internal.configentry.Status + 52, // 82: hashicorp.consul.internal.configentry.Status.Conditions:type_name -> hashicorp.consul.internal.configentry.Condition + 55, // 83: hashicorp.consul.internal.configentry.Condition.Resource:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 92, // 84: hashicorp.consul.internal.configentry.Condition.LastTransitionTime:type_name -> google.protobuf.Timestamp + 5, // 85: hashicorp.consul.internal.configentry.APIGatewayListener.Protocol:type_name -> hashicorp.consul.internal.configentry.APIGatewayListenerProtocol + 54, // 86: hashicorp.consul.internal.configentry.APIGatewayListener.TLS:type_name -> hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration + 55, // 87: hashicorp.consul.internal.configentry.APIGatewayTLSConfiguration.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 89, // 88: hashicorp.consul.internal.configentry.ResourceReference.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 83, // 89: hashicorp.consul.internal.configentry.BoundAPIGateway.Meta:type_name -> hashicorp.consul.internal.configentry.BoundAPIGateway.MetaEntry + 57, // 90: hashicorp.consul.internal.configentry.BoundAPIGateway.Listeners:type_name -> hashicorp.consul.internal.configentry.BoundAPIGatewayListener + 55, // 91: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Certificates:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 55, // 92: hashicorp.consul.internal.configentry.BoundAPIGatewayListener.Routes:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 84, // 93: hashicorp.consul.internal.configentry.InlineCertificate.Meta:type_name -> hashicorp.consul.internal.configentry.InlineCertificate.MetaEntry + 85, // 94: hashicorp.consul.internal.configentry.HTTPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.HTTPRoute.MetaEntry + 55, // 95: hashicorp.consul.internal.configentry.HTTPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 60, // 96: hashicorp.consul.internal.configentry.HTTPRoute.Rules:type_name -> hashicorp.consul.internal.configentry.HTTPRouteRule + 51, // 97: hashicorp.consul.internal.configentry.HTTPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status + 65, // 98: hashicorp.consul.internal.configentry.HTTPRouteRule.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters + 61, // 99: hashicorp.consul.internal.configentry.HTTPRouteRule.Matches:type_name -> hashicorp.consul.internal.configentry.HTTPMatch + 68, // 100: hashicorp.consul.internal.configentry.HTTPRouteRule.Services:type_name -> hashicorp.consul.internal.configentry.HTTPService + 62, // 101: hashicorp.consul.internal.configentry.HTTPMatch.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatch + 6, // 102: hashicorp.consul.internal.configentry.HTTPMatch.Method:type_name -> hashicorp.consul.internal.configentry.HTTPMatchMethod + 63, // 103: hashicorp.consul.internal.configentry.HTTPMatch.Path:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatch + 64, // 104: hashicorp.consul.internal.configentry.HTTPMatch.Query:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatch + 7, // 105: hashicorp.consul.internal.configentry.HTTPHeaderMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderMatchType + 8, // 106: hashicorp.consul.internal.configentry.HTTPPathMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPPathMatchType + 9, // 107: hashicorp.consul.internal.configentry.HTTPQueryMatch.Match:type_name -> hashicorp.consul.internal.configentry.HTTPQueryMatchType + 67, // 108: hashicorp.consul.internal.configentry.HTTPFilters.Headers:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter + 66, // 109: hashicorp.consul.internal.configentry.HTTPFilters.URLRewrite:type_name -> hashicorp.consul.internal.configentry.URLRewrite + 86, // 110: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Add:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.AddEntry + 87, // 111: hashicorp.consul.internal.configentry.HTTPHeaderFilter.Set:type_name -> hashicorp.consul.internal.configentry.HTTPHeaderFilter.SetEntry + 65, // 112: hashicorp.consul.internal.configentry.HTTPService.Filters:type_name -> hashicorp.consul.internal.configentry.HTTPFilters + 89, // 113: hashicorp.consul.internal.configentry.HTTPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 88, // 114: hashicorp.consul.internal.configentry.TCPRoute.Meta:type_name -> hashicorp.consul.internal.configentry.TCPRoute.MetaEntry + 55, // 115: hashicorp.consul.internal.configentry.TCPRoute.Parents:type_name -> hashicorp.consul.internal.configentry.ResourceReference + 70, // 116: hashicorp.consul.internal.configentry.TCPRoute.Services:type_name -> hashicorp.consul.internal.configentry.TCPService + 51, // 117: hashicorp.consul.internal.configentry.TCPRoute.Status:type_name -> hashicorp.consul.internal.configentry.Status + 89, // 118: hashicorp.consul.internal.configentry.TCPService.EnterpriseMeta:type_name -> hashicorp.consul.internal.common.EnterpriseMeta + 18, // 119: hashicorp.consul.internal.configentry.ServiceResolver.SubsetsEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverSubset + 20, // 120: hashicorp.consul.internal.configentry.ServiceResolver.FailoverEntry.value:type_name -> hashicorp.consul.internal.configentry.ServiceResolverFailover + 121, // [121:121] is the sub-list for method output_type + 121, // [121:121] is the sub-list for method input_type + 121, // [121:121] is the sub-list for extension type_name + 121, // [121:121] is the sub-list for extension extendee + 0, // [0:121] is the sub-list for field type_name } func init() { file_proto_pbconfigentry_config_entry_proto_init() } diff --git a/proto/pbconfigentry/config_entry.proto b/proto/pbconfigentry/config_entry.proto index 3fb6b649c1a0..476842fb6714 100644 --- a/proto/pbconfigentry/config_entry.proto +++ b/proto/pbconfigentry/config_entry.proto @@ -121,6 +121,8 @@ message ServiceResolver { google.protobuf.Duration ConnectTimeout = 5; LoadBalancer LoadBalancer = 6; map Meta = 7; + // mog: func-to=structs.DurationFromProto func-from=structs.DurationToProto + google.protobuf.Duration RequestTimeout = 8; } // mog annotation: diff --git a/website/content/docs/connect/config-entries/service-resolver.mdx b/website/content/docs/connect/config-entries/service-resolver.mdx index 9730beb132e9..89147a24af3c 100644 --- a/website/content/docs/connect/config-entries/service-resolver.mdx +++ b/website/content/docs/connect/config-entries/service-resolver.mdx @@ -440,6 +440,12 @@ spec: description: 'The timeout for establishing new network connections to this service. The default unit of time is `ns`.', }, + { + name: 'RequestTimeout', + type: 'duration: 0s', + description: + 'The timeout for receiving an HTTP response from this service before the connection is terminated. If unspecified or 0, the default of 15s is used. The default unit of time is `ns`.', + }, { name: 'DefaultSubset', type: 'string: ""', From 9c409797e6a7dde56806335bdedc5a78d3d7a888 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 10:45:47 -0600 Subject: [PATCH 078/421] Backport of Fix issue where terminating gateway service resolvers weren't properly cleaned up into release/1.15.x (#16521) * backport of commit e14b4301fad9b41a7fabda0e3b1f00fb1ca09995 * backport of commit 525501337d949fc1110a21f080cc2c1854ffc761 * backport of commit b1b2abc14a473864461a5dccab2911cb84f46225 * backport of commit ecaeff26aa1a2723a3b6239c08a125acf48731c9 --------- Co-authored-by: Andrew Stucki --- .changelog/16498.txt | 3 +++ agent/proxycfg/state_test.go | 22 +++++++++++++++++++++ agent/proxycfg/terminating_gateway.go | 7 ++++++- test/integration/connect/envoy/helpers.bash | 18 +++++++++++++++-- 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 .changelog/16498.txt diff --git a/.changelog/16498.txt b/.changelog/16498.txt new file mode 100644 index 000000000000..cdb045d67c9a --- /dev/null +++ b/.changelog/16498.txt @@ -0,0 +1,3 @@ +```release-note:bug +proxycfg: fix a bug where terminating gateways were not cleaning up deleted service resolvers for their referenced services +``` diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 894f2bd4795a..bdb2e3c07bc7 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -2093,6 +2093,28 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Equal(t, dbResolver.Entry, snap.TerminatingGateway.ServiceResolvers[db]) }, }, + { + requiredWatches: map[string]verifyWatchRequest{ + "service-resolver:" + db.String(): genVerifyResolverWatch("db", "dc1", structs.ServiceResolver), + }, + events: []UpdateEvent{ + { + CorrelationID: "service-resolver:" + db.String(), + Result: &structs.ConfigEntryResponse{ + Entry: nil, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "gateway with service list is valid") + // Finally ensure we cleaned up the resolver + require.Equal(t, []structs.ServiceName{db}, snap.TerminatingGateway.ValidServices()) + + require.False(t, snap.TerminatingGateway.ServiceResolversSet[db]) + require.Nil(t, snap.TerminatingGateway.ServiceResolvers[db]) + }, + }, { events: []UpdateEvent{ { diff --git a/agent/proxycfg/terminating_gateway.go b/agent/proxycfg/terminating_gateway.go index cb371ae2bf02..483c79f91ddc 100644 --- a/agent/proxycfg/terminating_gateway.go +++ b/agent/proxycfg/terminating_gateway.go @@ -354,8 +354,13 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv // There should only ever be one entry for a service resolver within a namespace if resolver, ok := resp.Entry.(*structs.ServiceResolverConfigEntry); ok { snap.TerminatingGateway.ServiceResolvers[sn] = resolver + snap.TerminatingGateway.ServiceResolversSet[sn] = true + } else { + // we likely have a deleted service resolver, and our cast is a nil + // cast, so clear this out + delete(snap.TerminatingGateway.ServiceResolvers, sn) + snap.TerminatingGateway.ServiceResolversSet[sn] = false } - snap.TerminatingGateway.ServiceResolversSet[sn] = true case strings.HasPrefix(u.CorrelationID, serviceIntentionsIDPrefix): resp, ok := u.Result.(structs.Intentions) diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index c7746d9260db..65bbe3b0070f 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -383,15 +383,27 @@ function assert_upstream_has_endpoints_in_status_once { GOT_COUNT=$(get_upstream_endpoint_in_status_count $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS) + echo "GOT: $GOT_COUNT" [ "$GOT_COUNT" -eq $EXPECT_COUNT ] } +function assert_upstream_missing_once { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + + run get_upstream_endpoint $HOSTPORT $CLUSTER_NAME + [ "$status" -eq 0 ] + echo "$output" + [ "" == "$output" ] +} + function assert_upstream_missing { local HOSTPORT=$1 local CLUSTER_NAME=$2 - run retry_default get_upstream_endpoint $HOSTPORT $CLUSTER_NAME + run retry_long assert_upstream_missing_once $HOSTPORT $CLUSTER_NAME echo "OUTPUT: $output $status" - [ "" == "$output" ] + + [ "$status" -eq 0 ] } function assert_upstream_has_endpoints_in_status { @@ -400,6 +412,8 @@ function assert_upstream_has_endpoints_in_status { local HEALTH_STATUS=$3 local EXPECT_COUNT=$4 run retry_long assert_upstream_has_endpoints_in_status_once $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS $EXPECT_COUNT + echo "$output" + [ "$status" -eq 0 ] } From c07b931987356f22fc0589fceb40f79d815b9c58 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 12:52:42 -0600 Subject: [PATCH 079/421] backport of commit f7636be2118590702a6e35e4c04568b8793a3379 (#16524) Co-authored-by: Michael Wilkerson --- command/connect/envoy/envoy.go | 69 ++++++++++++++++++++++------- command/connect/envoy/envoy_test.go | 33 ++++++++++---- 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index c265c0ba9cd8..102c16494db3 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -503,15 +503,15 @@ func (c *cmd) run(args []string) int { return 1 } - ok, err := checkEnvoyVersionCompatibility(v, xdscommon.UnsupportedEnvoyVersions) + ec, err := checkEnvoyVersionCompatibility(v, xdscommon.UnsupportedEnvoyVersions) if err != nil { c.UI.Warn("There was an error checking the compatibility of the envoy version: " + err.Error()) - } else if !ok { + } else if !ec.isCompatible { c.UI.Error(fmt.Sprintf("Envoy version %s is not supported. If there is a reason you need to use "+ "this version of envoy use the ignore-envoy-compatibility flag. Using an unsupported version of Envoy "+ "is not recommended and your experience may vary. For more information on compatibility "+ - "see https://developer.hashicorp.com/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent", v)) + "see https://developer.hashicorp.com/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent", ec.versionIncompatible)) return 1 } } @@ -976,34 +976,73 @@ Usage: consul connect envoy [options] [-- pass-through options] ` ) -func checkEnvoyVersionCompatibility(envoyVersion string, unsupportedList []string) (bool, error) { - // Now compare the versions to the list of supported versions +type envoyCompat struct { + isCompatible bool + versionIncompatible string +} + +func checkEnvoyVersionCompatibility(envoyVersion string, unsupportedList []string) (envoyCompat, error) { v, err := version.NewVersion(envoyVersion) if err != nil { - return false, err + return envoyCompat{}, err } var cs strings.Builder - // Add one to the max minor version so that we accept all patches + // If there is a list of unsupported versions, build the constraint string, + // this will detect exactly unsupported versions + if len(unsupportedList) > 0 { + for i, s := range unsupportedList { + if i == 0 { + cs.WriteString(fmt.Sprintf("!= %s", s)) + } else { + cs.WriteString(fmt.Sprintf(", != %s", s)) + } + } + + constraints, err := version.NewConstraint(cs.String()) + if err != nil { + return envoyCompat{}, err + } + + if c := constraints.Check(v); !c { + return envoyCompat{ + isCompatible: c, + versionIncompatible: envoyVersion, + }, nil + } + } + + // Next build the constraint string using the bounds, make sure that we are less than but not equal to + // maxSupported since we will add 1. Need to add one to the max minor version so that we accept all patches splitS := strings.Split(xdscommon.GetMaxEnvoyMinorVersion(), ".") minor, err := strconv.Atoi(splitS[1]) if err != nil { - return false, err + return envoyCompat{}, err } minor++ maxSupported := fmt.Sprintf("%s.%d", splitS[0], minor) - // Build the constraint string, make sure that we are less than but not equal to maxSupported since we added 1 + cs.Reset() cs.WriteString(fmt.Sprintf(">= %s, < %s", xdscommon.GetMinEnvoyMinorVersion(), maxSupported)) - for _, s := range unsupportedList { - cs.WriteString(fmt.Sprintf(", != %s", s)) - } - constraints, err := version.NewConstraint(cs.String()) if err != nil { - return false, err + return envoyCompat{}, err } - return constraints.Check(v), nil + if c := constraints.Check(v); !c { + return envoyCompat{ + isCompatible: c, + versionIncompatible: replacePatchVersionWithX(envoyVersion), + }, nil + } + + return envoyCompat{isCompatible: true}, nil +} + +func replacePatchVersionWithX(version string) string { + // Strip off the patch and append x to convey that the constraint is on the minor version and not the patch + // itself + a := strings.Split(version, ".") + return fmt.Sprintf("%s.%s.x", a[0], a[1]) } diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 223eb2e130ca..09c02c91c094 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -1696,50 +1696,65 @@ func TestCheckEnvoyVersionCompatibility(t *testing.T) { name string envoyVersion string unsupportedList []string - expectedSupport bool + expectedCompat envoyCompat isErrorExpected bool }{ { name: "supported-using-proxy-support-defined", envoyVersion: xdscommon.EnvoyVersions[1], unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: true, + expectedCompat: envoyCompat{ + isCompatible: true, + }, }, { name: "supported-at-max", envoyVersion: xdscommon.GetMaxEnvoyMinorVersion(), unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: true, + expectedCompat: envoyCompat{ + isCompatible: true, + }, }, { name: "supported-patch-higher", envoyVersion: addNPatchVersion(xdscommon.EnvoyVersions[0], 1), unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: true, + expectedCompat: envoyCompat{ + isCompatible: true, + }, }, { name: "not-supported-minor-higher", envoyVersion: addNMinorVersion(xdscommon.EnvoyVersions[0], 1), unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: false, + expectedCompat: envoyCompat{ + isCompatible: false, + versionIncompatible: replacePatchVersionWithX(addNMinorVersion(xdscommon.EnvoyVersions[0], 1)), + }, }, { name: "not-supported-minor-lower", envoyVersion: addNMinorVersion(xdscommon.EnvoyVersions[len(xdscommon.EnvoyVersions)-1], -1), unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: false, + expectedCompat: envoyCompat{ + isCompatible: false, + versionIncompatible: replacePatchVersionWithX(addNMinorVersion(xdscommon.EnvoyVersions[len(xdscommon.EnvoyVersions)-1], -1)), + }, }, { name: "not-supported-explicitly-unsupported-version", envoyVersion: addNPatchVersion(xdscommon.EnvoyVersions[0], 1), unsupportedList: []string{"1.23.1", addNPatchVersion(xdscommon.EnvoyVersions[0], 1)}, - expectedSupport: false, + expectedCompat: envoyCompat{ + isCompatible: false, + versionIncompatible: addNPatchVersion(xdscommon.EnvoyVersions[0], 1), + }, }, { name: "error-bad-input", envoyVersion: "1.abc.3", unsupportedList: xdscommon.UnsupportedEnvoyVersions, - expectedSupport: false, + expectedCompat: envoyCompat{}, isErrorExpected: true, }, } @@ -1752,7 +1767,7 @@ func TestCheckEnvoyVersionCompatibility(t *testing.T) { } else { assert.NoError(t, err) } - assert.Equal(t, tc.expectedSupport, actual) + assert.Equal(t, tc.expectedCompat, actual) }) } } From b3f0b10b42f5369de1ae6bc9867be0aea0da3217 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 13:43:26 -0600 Subject: [PATCH 080/421] Backport of fixed broken links associated with cluster peering updates into release/1.15.x (#16527) * backport of commit 39a967064c030d94e3e81a4f99828e9ba7f6a223 * backport of commit 14364abd1dfbcd8f7cd56029d6378850094e61f2 * missing redirect entry --------- Co-authored-by: trujillo-adam --- .../api-gateway/usage/route-to-peered-services.mdx | 4 ++-- .../content/docs/connect/cluster-peering/index.mdx | 2 +- .../content/docs/k8s/annotations-and-labels.mdx | 4 ++-- .../cluster-peering/usage/establish-peering.mdx | 2 +- website/content/docs/k8s/crds/index.mdx | 4 ++-- .../docs/release-notes/consul-k8s/v0_47_x.mdx | 2 +- website/redirects.js | 14 ++++++++++---- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx index 33bc54cdb48d..fe8ca69732cc 100644 --- a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx +++ b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx @@ -12,8 +12,8 @@ This topic describes how to configure Consul API Gateway to route traffic to ser 1. Consul 1.14 or later 1. Verify that the [requirements](/consul/docs/api-gateway/tech-specs) have been met. 1. Verify that the Consul API Gateway CRDs and controller have been installed and applied. Refer to [Installation](/consul/docs/api-gateway/install) for details. -1. A peering connection must already be established between Consul clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for instructions. -1. The Consul service that you want to route traffic to must be exported to the cluster containing your `Gateway`. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for instructions. +1. A peering connection must already be established between Consul clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/tech-specs) for instructions. +1. The Consul service that you want to route traffic to must be exported to the cluster containing your `Gateway`. Refer to [Cluster Peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/tech-specs) for instructions. 1. A `ServiceResolver` for the Consul service you want to route traffic to must be created in the cluster that contains your `Gateway`. Refer to [Service Resolver Configuration Entry](/consul/docs/connect/config-entries/service-resolver) for instructions. ## Configuration diff --git a/website/content/docs/connect/cluster-peering/index.mdx b/website/content/docs/connect/cluster-peering/index.mdx index aeb940d638c1..f8e1c18caf3a 100644 --- a/website/content/docs/connect/cluster-peering/index.mdx +++ b/website/content/docs/connect/cluster-peering/index.mdx @@ -57,7 +57,7 @@ The following resources are available to help you use Consul's cluster peering f **Usage documentation:** -- [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering) +- [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering) - [Manage cluster peering connections](/consul/docs/connect/cluster-peering/usage/manage-connections) - [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management) diff --git a/website/content/docs/k8s/annotations-and-labels.mdx b/website/content/docs/k8s/annotations-and-labels.mdx index d6ea11d4f1dc..d51b591ac863 100644 --- a/website/content/docs/k8s/annotations-and-labels.mdx +++ b/website/content/docs/k8s/annotations-and-labels.mdx @@ -75,7 +75,7 @@ The following Kubernetes resource annotations could be used on a pod to control - Unlabeled: Use the unlabeled annotation format to specify a service name, Consul Enterprise namespaces and partitions, and - datacenters. To use [cluster peering](/consul/docs/connect/cluster-peering/k8s) with upstreams, use the following + datacenters. To use [cluster peering](/consul/docs/k8s/connect/cluster-peering/k8s-tech-specs) with upstreams, use the following labeled format. - Service name: Place the service name at the beginning of the annotation to specify the upstream service. You can also append the datacenter where the service is deployed (optional). @@ -98,7 +98,7 @@ The following Kubernetes resource annotations could be used on a pod to control - Admin partitions (requires Consul Enterprise 1.11+): Upstream services may be running in a different partition. You must specify the namespace when specifying a partition. Place the partition name after the namespace. If you specify the name of the datacenter (optional), it must be the local datacenter. Communicating across partitions using this method is only supported within a datacenter. For cross partition communication across datacenters, refer to [cluster - peering](/consul/docs/connect/cluster-peering/k8s). + peering](/consul/docs/k8s/connect/cluster-peering/k8s-tech-specs). ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].[service-namespace].[service-partition]:[port]:[optional datacenter]" diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx index 3c85706cc626..19e504b95d68 100644 --- a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx +++ b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx @@ -18,7 +18,7 @@ The overall process for establishing a cluster peering connection consists of th Cluster peering between services cannot be established until all four steps are complete. -For general guidance for establishing cluster peering connections, refer to [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering). +For general guidance for establishing cluster peering connections, refer to [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering). ## Prerequisites diff --git a/website/content/docs/k8s/crds/index.mdx b/website/content/docs/k8s/crds/index.mdx index 342fe1aaa4c4..6a68960a04df 100644 --- a/website/content/docs/k8s/crds/index.mdx +++ b/website/content/docs/k8s/crds/index.mdx @@ -16,8 +16,8 @@ You can specify the following values in the `kind` field. Click on a configurati - [`Mesh`](/consul/docs/connect/config-entries/mesh) - [`ExportedServices`](/consul/docs/connect/config-entries/exported-services) -- [`PeeringAcceptor`](/consul/docs/connect/cluster-peering/k8s#peeringacceptor) -- [`PeeringDialer`](/consul/docs/connect/cluster-peering/k8s#peeringdialer) +- [`PeeringAcceptor`](/consul/docs/k8s/connect/cluster-peering/tech-specs#peeringacceptor) +- [`PeeringDialer`](/consul/docs/k8s/connect/cluster-peering/tech-specs#peeringdialer) - [`ProxyDefaults`](/consul/docs/connect/config-entries/proxy-defaults) - [`ServiceDefaults`](/consul/docs/connect/config-entries/service-defaults) - [`ServiceSplitter`](/consul/docs/connect/config-entries/service-splitter) diff --git a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx index 3d266b133505..8f185bb6f372 100644 --- a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx @@ -9,7 +9,7 @@ description: >- ## Release Highlights -- **Cluster Peering (Beta)**: This release introduces support for Cluster Peering, which allows service connectivity between two independent clusters. Enabling peering will deploy the peering controllers and PeeringAcceptor and PeeringDialer CRDs. The new CRDs are used to establish a peering connection between two clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for full instructions on using Cluster Peering on Kubernetes. +- **Cluster Peering (Beta)**: This release introduces support for cluster peering, which allows service connectivity between two independent clusters. When you enable cluster peering, Consul deploys the peering controllers and `PeeringAcceptor` and `PeeringDialer` CRDs. The new CRDs are used to establish a peering connection between two clusters. Refer to [Cluster Peering Overview](/consul/docs/connect/cluster-peering) for full instructions on using Cluster Peering on Kubernetes. - **Envoy Proxy Debugging CLI Commands**: This release introduces new commands to quickly identify proxies and troubleshoot Envoy proxies for sidecars and gateways. * Add `consul-k8s proxy list` command for displaying pods running Envoy managed by Consul. diff --git a/website/redirects.js b/website/redirects.js index 74caeb5e3885..3e4625016505 100644 --- a/website/redirects.js +++ b/website/redirects.js @@ -6,14 +6,20 @@ module.exports = [ { - source: '/docs/connect/cluster-peering/create-manage-peering', + source: '/consul/docs/connect/cluster-peering/create-manage-peering', destination: - '/docs/connect/cluster-peering/usage/establish-cluster-peering', + '/consul/docs/connect/cluster-peering/usage/establish-cluster-peering', permanent: true, }, { - source: '/docs/connect/cluster-peering/k8s', - destination: '/docs/k8s/connect/cluster-peering/k8s-tech-specs', + source: '/consul/docs/connect/cluster-peering/usage/establish-peering', + destination: + '/consul/docs/connect/cluster-peering/usage/establish-cluster-peering', + permanent: true, + }, + { + source: '/consul/docs/connect/cluster-peering/k8s', + destination: '/consul/docs/k8s/connect/cluster-peering/tech-specs', permanent: true, }, ] From 0f139f243967aa7b094dec9a15a2b68c263ace27 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 13:44:50 -0600 Subject: [PATCH 081/421] Backport of Fix resolution of service resolvers with subsets for external upstreams into release/1.15.x (#16525) * backport of commit 892d389d9b177c90ffa886bf3bc5d17a87556cec * backport of commit 8a2468d6b541de31d23f6d4d9ba28b4560ceea68 * backport of commit f56894fdc1ef8cbe824d06ad3aeaf84382822dc0 * backport of commit ced73fc2ce09a1b488c3744ce96686d2593992d8 --------- Co-authored-by: Andrew Stucki --- .changelog/16499.txt | 3 ++ agent/rpcclient/health/view.go | 24 ++++++++++--- agent/rpcclient/health/view_test.go | 36 +++++++++++++++++++ .../capture.sh | 1 + .../service_s3.hcl | 17 +++++++++ .../case-terminating-gateway-subsets/setup.sh | 1 + .../case-terminating-gateway-subsets/vars.sh | 1 + .../verify.bats | 4 +++ test/integration/connect/envoy/helpers.bash | 33 +++++++++++++++++ 9 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 .changelog/16499.txt create mode 100644 test/integration/connect/envoy/case-terminating-gateway-subsets/service_s3.hcl diff --git a/.changelog/16499.txt b/.changelog/16499.txt new file mode 100644 index 000000000000..4bd50db47e8a --- /dev/null +++ b/.changelog/16499.txt @@ -0,0 +1,3 @@ +```release-note:bug +mesh: Fix resolution of service resolvers with subsets for external upstreams +``` diff --git a/agent/rpcclient/health/view.go b/agent/rpcclient/health/view.go index fd19cb4a001b..ce32e801c335 100644 --- a/agent/rpcclient/health/view.go +++ b/agent/rpcclient/health/view.go @@ -50,8 +50,10 @@ func NewHealthView(req structs.ServiceSpecificRequest) (*HealthView, error) { return nil, err } return &HealthView{ - state: make(map[string]structs.CheckServiceNode), - filter: fe, + state: make(map[string]structs.CheckServiceNode), + filter: fe, + connect: req.Connect, + kind: req.ServiceKind, }, nil } @@ -61,8 +63,10 @@ func NewHealthView(req structs.ServiceSpecificRequest) (*HealthView, error) { // (IndexedCheckServiceNodes) and update it in place for each event - that // involves re-sorting each time etc. though. type HealthView struct { - state map[string]structs.CheckServiceNode - filter filterEvaluator + connect bool + kind structs.ServiceKind + state map[string]structs.CheckServiceNode + filter filterEvaluator } // Update implements View @@ -84,6 +88,13 @@ func (s *HealthView) Update(events []*pbsubscribe.Event) error { if csn == nil { return errors.New("check service node was unexpectedly nil") } + + // check if we intentionally need to skip the filter + if s.skipFilter(csn) { + s.state[id] = *csn + continue + } + passed, err := s.filter.Evaluate(*csn) if err != nil { return err @@ -100,6 +111,11 @@ func (s *HealthView) Update(events []*pbsubscribe.Event) error { return nil } +func (s *HealthView) skipFilter(csn *structs.CheckServiceNode) bool { + // we only do this for connect-enabled services that need to be routed through a terminating gateway + return s.kind == "" && s.connect && csn.Service.Kind == structs.ServiceKindTerminatingGateway +} + type filterEvaluator interface { Evaluate(datum interface{}) (bool, error) } diff --git a/agent/rpcclient/health/view_test.go b/agent/rpcclient/health/view_test.go index 8fcb50da3396..f1d2cd0869d3 100644 --- a/agent/rpcclient/health/view_test.go +++ b/agent/rpcclient/health/view_test.go @@ -941,3 +941,39 @@ func TestNewFilterEvaluator(t *testing.T) { }) } } + +func TestHealthView_SkipFilteringTerminatingGateways(t *testing.T) { + view, err := NewHealthView(structs.ServiceSpecificRequest{ + ServiceName: "name", + Connect: true, + QueryOptions: structs.QueryOptions{ + Filter: "Service.Meta.version == \"v1\"", + }, + }) + require.NoError(t, err) + + err = view.Update([]*pbsubscribe.Event{{ + Index: 1, + Payload: &pbsubscribe.Event_ServiceHealth{ + ServiceHealth: &pbsubscribe.ServiceHealthUpdate{ + Op: pbsubscribe.CatalogOp_Register, + CheckServiceNode: &pbservice.CheckServiceNode{ + Service: &pbservice.NodeService{ + Kind: structs.TerminatingGateway, + Service: "name", + Address: "127.0.0.1", + Port: 8443, + }, + }, + }, + }, + }}) + require.NoError(t, err) + + node, ok := (view.Result(1)).(*structs.IndexedCheckServiceNodes) + require.True(t, ok) + + require.Len(t, node.Nodes, 1) + require.Equal(t, "127.0.0.1", node.Nodes[0].Service.Address) + require.Equal(t, 8443, node.Nodes[0].Service.Port) +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-subsets/capture.sh b/test/integration/connect/envoy/case-terminating-gateway-subsets/capture.sh index 2ef0c41a215e..261bf4e29a6c 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-subsets/capture.sh +++ b/test/integration/connect/envoy/case-terminating-gateway-subsets/capture.sh @@ -2,3 +2,4 @@ snapshot_envoy_admin localhost:20000 terminating-gateway primary || true snapshot_envoy_admin localhost:19000 s1 primary || true +snapshot_envoy_admin localhost:19001 s3 primary || true diff --git a/test/integration/connect/envoy/case-terminating-gateway-subsets/service_s3.hcl b/test/integration/connect/envoy/case-terminating-gateway-subsets/service_s3.hcl new file mode 100644 index 000000000000..eb84c578ee9e --- /dev/null +++ b/test/integration/connect/envoy/case-terminating-gateway-subsets/service_s3.hcl @@ -0,0 +1,17 @@ +services { + id = "s3" + name = "s3" + port = 8184 + connect { + sidecar_service { + proxy { + upstreams = [ + { + destination_name = "s2" + local_bind_port = 8185 + } + ] + } + } + } +} diff --git a/test/integration/connect/envoy/case-terminating-gateway-subsets/setup.sh b/test/integration/connect/envoy/case-terminating-gateway-subsets/setup.sh index fdd49572ba8d..57b85c74a6b2 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-subsets/setup.sh +++ b/test/integration/connect/envoy/case-terminating-gateway-subsets/setup.sh @@ -38,4 +38,5 @@ register_services primary # terminating gateway will act as s2's proxy gen_envoy_bootstrap s1 19000 +gen_envoy_bootstrap s3 19001 gen_envoy_bootstrap terminating-gateway 20000 primary true diff --git a/test/integration/connect/envoy/case-terminating-gateway-subsets/vars.sh b/test/integration/connect/envoy/case-terminating-gateway-subsets/vars.sh index 9e52629b8bed..d4a4d75bdd88 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-subsets/vars.sh +++ b/test/integration/connect/envoy/case-terminating-gateway-subsets/vars.sh @@ -4,5 +4,6 @@ export REQUIRED_SERVICES=" s1 s1-sidecar-proxy s2-v1 +s3 s3-sidecar-proxy terminating-gateway-primary " diff --git a/test/integration/connect/envoy/case-terminating-gateway-subsets/verify.bats b/test/integration/connect/envoy/case-terminating-gateway-subsets/verify.bats index 64a2499e3575..028ddea85ade 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-subsets/verify.bats +++ b/test/integration/connect/envoy/case-terminating-gateway-subsets/verify.bats @@ -38,3 +38,7 @@ load helpers assert_envoy_metric_at_least 127.0.0.1:20000 "v1.s2.default.primary.*cx_total" 1 } +@test "terminating-gateway is used for the upstream connection of the proxy" { + # make sure we resolve the terminating gateway as endpoint for the upstream + assert_upstream_has_endpoint_port 127.0.0.1:19001 "v1.s2" 8443 +} diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 65bbe3b0070f..a650f4ee29ce 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -361,6 +361,39 @@ function get_upstream_endpoint { | select(.name|startswith(\"${CLUSTER_NAME}\"))" } +function get_upstream_endpoint_port { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + run curl -s -f "http://${HOSTPORT}/clusters?format=json" + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output " +.cluster_statuses[] +| select(.name|startswith(\"${CLUSTER_NAME}\")) +| [.host_statuses[].address.socket_address.port_value] +| [select(.[] == ${PORT_VALUE})] +| length" +} + +function assert_upstream_has_endpoint_port_once { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + + GOT_COUNT=$(get_upstream_endpoint_port $HOSTPORT $CLUSTER_NAME $PORT_VALUE) + + [ "$GOT_COUNT" -eq 1 ] +} + +function assert_upstream_has_endpoint_port { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + + run retry_long assert_upstream_has_endpoint_port_once $HOSTPORT $CLUSTER_NAME $PORT_VALUE + [ "$status" -eq 0 ] +} + function get_upstream_endpoint_in_status_count { local HOSTPORT=$1 local CLUSTER_NAME=$2 From 388fef8484b29fe1cb84ad176cd4b263eec2e4a4 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 15:00:25 -0600 Subject: [PATCH 082/421] Backport of proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher into release/1.15.x (#16529) * backport of commit 88e56eac02ab058c0ca5d34ebb47d394356c51ee * backport of commit 4fc870086d5dddc771d071bf78f8026c40067bcc * backport of commit 0d57b2a70cf723be0b83d4ab8993edf300d6ea58 * backport of commit c627e5ac39fab6681f225bb0a92cc19b26675667 * backport of commit 48f93514de17f23e642d9badb32e30cad8469483 * backport of commit 3b9fbcd800e60abab90efa33699f7ac400b488a7 * backport of commit a323375c30aecd24fe12a7e164daa905de9412e9 * backport of commit 70b54d501fc9311baac0d0f63fd67640a50d6a01 --------- Co-authored-by: R.B. Boyer Co-authored-by: R.B. Boyer <4903+rboyer@users.noreply.github.com> --- .changelog/16497.txt | 3 + agent/agent.go | 11 +- agent/config/builder.go | 1 + agent/config/runtime.go | 4 + agent/config/runtime_test.go | 13 +- .../TestRuntimeConfig_Sanitize.golden | 1 + agent/proxycfg-sources/local/sync.go | 14 ++ agent/proxycfg/manager.go | 2 +- agent/proxycfg/state.go | 32 +++- agent/proxycfg_test.go | 138 ++++++++++++++++++ agent/testagent.go | 3 + 11 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 .changelog/16497.txt create mode 100644 agent/proxycfg_test.go diff --git a/.changelog/16497.txt b/.changelog/16497.txt new file mode 100644 index 000000000000..3aa3633ac3a6 --- /dev/null +++ b/.changelog/16497.txt @@ -0,0 +1,3 @@ +```release-note:bug +proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher +``` diff --git a/agent/agent.go b/agent/agent.go index bd8c701ca794..9c85f065ec2b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -721,11 +721,12 @@ func (a *Agent) Start(ctx context.Context) error { go localproxycfg.Sync( &lib.StopChannelContext{StopCh: a.shutdownCh}, localproxycfg.SyncConfig{ - Manager: a.proxyConfig, - State: a.State, - Logger: a.proxyConfig.Logger.Named("agent-state"), - Tokens: a.baseDeps.Tokens, - NodeName: a.config.NodeName, + Manager: a.proxyConfig, + State: a.State, + Logger: a.proxyConfig.Logger.Named("agent-state"), + Tokens: a.baseDeps.Tokens, + NodeName: a.config.NodeName, + ResyncFrequency: a.config.LocalProxyConfigResyncInterval, }, ) diff --git a/agent/config/builder.go b/agent/config/builder.go index f682bf7b1425..5d697b027ee7 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1091,6 +1091,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { Watches: c.Watches, XDSUpdateRateLimit: limitVal(c.XDS.UpdateMaxPerSecond), AutoReloadConfigCoalesceInterval: 1 * time.Second, + LocalProxyConfigResyncInterval: 30 * time.Second, } rt.TLS, err = b.buildTLSConfig(rt, c.TLS) diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 627c1e564401..b0d9cf436e55 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -1475,6 +1475,10 @@ type RuntimeConfig struct { // AutoReloadConfigCoalesceInterval Coalesce Interval for auto reload config AutoReloadConfigCoalesceInterval time.Duration + // LocalProxyConfigResyncInterval is not a user-configurable value and exists + // here so that tests can use a smaller value. + LocalProxyConfigResyncInterval time.Duration + EnterpriseRuntimeConfig } diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 2954dee6700b..af1f5085bb56 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -5995,12 +5995,13 @@ func TestLoad_FullConfig(t *testing.T) { nodeEntMeta := structs.NodeEnterpriseMetaInDefaultPartition() expected := &RuntimeConfig{ // non-user configurable values - AEInterval: time.Minute, - CheckDeregisterIntervalMin: time.Minute, - CheckReapInterval: 30 * time.Second, - SegmentNameLimit: 64, - SyncCoordinateIntervalMin: 15 * time.Second, - SyncCoordinateRateTarget: 64, + AEInterval: time.Minute, + CheckDeregisterIntervalMin: time.Minute, + CheckReapInterval: 30 * time.Second, + SegmentNameLimit: 64, + SyncCoordinateIntervalMin: 15 * time.Second, + SyncCoordinateRateTarget: 64, + LocalProxyConfigResyncInterval: 30 * time.Second, Revision: "JNtPSav3", Version: "R909Hblt", diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index 75d216fabdb7..2c5b91c98a53 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -233,6 +233,7 @@ "KVMaxValueSize": 1234567800000000, "LeaveDrainTime": "0s", "LeaveOnTerm": false, + "LocalProxyConfigResyncInterval": "0s", "Logging": { "EnableSyslog": false, "LogFilePath": "", diff --git a/agent/proxycfg-sources/local/sync.go b/agent/proxycfg-sources/local/sync.go index c6cee8c61d15..5702d2f36841 100644 --- a/agent/proxycfg-sources/local/sync.go +++ b/agent/proxycfg-sources/local/sync.go @@ -2,6 +2,7 @@ package local import ( "context" + "time" "github.com/hashicorp/go-hclog" @@ -11,6 +12,8 @@ import ( "github.com/hashicorp/consul/agent/token" ) +const resyncFrequency = 30 * time.Second + const source proxycfg.ProxySource = "local" // SyncConfig contains the dependencies required by Sync. @@ -30,6 +33,10 @@ type SyncConfig struct { // Logger will be used to write log messages. Logger hclog.Logger + + // ResyncFrequency is how often to do a resync and recreate any terminated + // watches. + ResyncFrequency time.Duration } // Sync watches the agent's local state and registers/deregisters services with @@ -50,12 +57,19 @@ func Sync(ctx context.Context, cfg SyncConfig) { cfg.State.Notify(stateCh) defer cfg.State.StopNotify(stateCh) + var resyncCh <-chan time.Time for { sync(cfg) + if resyncCh == nil && cfg.ResyncFrequency > 0 { + resyncCh = time.After(cfg.ResyncFrequency) + } + select { case <-stateCh: // Wait for a state change. + case <-resyncCh: + resyncCh = nil case <-ctx.Done(): return } diff --git a/agent/proxycfg/manager.go b/agent/proxycfg/manager.go index c58268e7e039..d21ff4f1ea5b 100644 --- a/agent/proxycfg/manager.go +++ b/agent/proxycfg/manager.go @@ -158,7 +158,7 @@ func (m *Manager) Register(id ProxyID, ns *structs.NodeService, source ProxySour func (m *Manager) register(id ProxyID, ns *structs.NodeService, source ProxySource, token string, overwrite bool) error { state, ok := m.proxies[id] - if ok { + if ok && !state.stoppedRunning() { if state.source != source && !overwrite { // Registered by a different source, leave as-is. return nil diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index d312c3b4c10c..2347b04cc53b 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -83,10 +83,20 @@ type state struct { ch chan UpdateEvent snapCh chan ConfigSnapshot reqCh chan chan *ConfigSnapshot + doneCh chan struct{} rateLimiter *rate.Limiter } +func (s *state) stoppedRunning() bool { + select { + case <-s.doneCh: + return true + default: + return false + } +} + // failed returns whether run exited because a data source is in an // irrecoverable state. func (s *state) failed() bool { @@ -182,6 +192,7 @@ func newState(id ProxyID, ns *structs.NodeService, source ProxySource, token str ch: ch, snapCh: make(chan ConfigSnapshot, 1), reqCh: make(chan chan *ConfigSnapshot, 1), + doneCh: make(chan struct{}), rateLimiter: rateLimiter, }, nil } @@ -265,6 +276,9 @@ func (s *state) Watch() (<-chan ConfigSnapshot, error) { // Close discards the state and stops any long-running watches. func (s *state) Close(failed bool) error { + if s.stoppedRunning() { + return nil + } if s.cancel != nil { s.cancel() } @@ -314,6 +328,9 @@ func (s *state) run(ctx context.Context, snap *ConfigSnapshot) { } func (s *state) unsafeRun(ctx context.Context, snap *ConfigSnapshot) { + // Closing the done channel signals that this entire state is no longer + // going to be updated. + defer close(s.doneCh) // Close the channel we return from Watch when we stop so consumers can stop // watching and clean up their goroutines. It's important we do this here and // not in Close since this routine sends on this chan and so might panic if it @@ -429,9 +446,20 @@ func (s *state) unsafeRun(ctx context.Context, snap *ConfigSnapshot) { func (s *state) CurrentSnapshot() *ConfigSnapshot { // Make a chan for the response to be sent on ch := make(chan *ConfigSnapshot, 1) - s.reqCh <- ch + + select { + case <-s.doneCh: + return nil + case s.reqCh <- ch: + } + // Wait for the response - return <-ch + select { + case <-s.doneCh: + return nil + case resp := <-ch: + return resp + } } // Changed returns whether or not the passed NodeService has had any of the diff --git a/agent/proxycfg_test.go b/agent/proxycfg_test.go new file mode 100644 index 000000000000..18a5c586245c --- /dev/null +++ b/agent/proxycfg_test.go @@ -0,0 +1,138 @@ +package agent + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent/grpc-external/limiter" + "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/testrpc" +) + +func TestAgent_local_proxycfg(t *testing.T) { + a := NewTestAgent(t, TestACLConfig()) + defer a.Shutdown() + + testrpc.WaitForLeader(t, a.RPC, "dc1") + + token := generateUUID() + + svc := &structs.NodeService{ + ID: "db", + Service: "db", + Port: 5000, + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + } + require.NoError(t, a.State.AddServiceWithChecks(svc, nil, token, true)) + + proxy := &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "db-sidecar-proxy", + Service: "db-sidecar-proxy", + Port: 5000, + // Set this internal state that we expect sidecar registrations to have. + LocallyRegisteredAsSidecar: true, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "db", + Upstreams: structs.TestUpstreams(t), + }, + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + } + require.NoError(t, a.State.AddServiceWithChecks(proxy, nil, token, true)) + + // This is a little gross, but this gives us the layered pair of + // local/catalog sources for now. + cfg := a.xdsServer.CfgSrc + + var ( + timer = time.After(100 * time.Millisecond) + timerFired = false + finalTimer <-chan time.Time + ) + + var ( + firstTime = true + ch <-chan *proxycfg.ConfigSnapshot + stc limiter.SessionTerminatedChan + cancel proxycfg.CancelFunc + ) + defer func() { + if cancel != nil { + cancel() + } + }() + for { + if ch == nil { + // Sign up for a stream of config snapshots, in the same manner as the xds server. + sid := proxy.CompoundServiceID() + + if firstTime { + firstTime = false + } else { + t.Logf("re-creating watch") + } + + // Prior to fixes in https://github.com/hashicorp/consul/pull/16497 + // this call to Watch() would deadlock. + var err error + ch, stc, cancel, err = cfg.Watch(sid, a.config.NodeName, token) + require.NoError(t, err) + } + select { + case <-stc: + t.Fatal("session unexpectedly terminated") + case snap, ok := <-ch: + if !ok { + t.Logf("channel is closed") + cancel() + ch, stc, cancel = nil, nil, nil + continue + } + require.NotNil(t, snap) + if !timerFired { + t.Fatal("should not have gotten snapshot until after we manifested the token") + } + return + case <-timer: + timerFired = true + finalTimer = time.After(1 * time.Second) + + // This simulates the eventual consistency of a token + // showing up on a server after it's creation by + // pre-creating the UUID and later using that as the + // initial SecretID for a real token. + gotToken := testWriteToken(t, a, &api.ACLToken{ + AccessorID: generateUUID(), + SecretID: token, + Description: "my token", + ServiceIdentities: []*api.ACLServiceIdentity{{ + ServiceName: "db", + }}, + }) + require.Equal(t, token, gotToken) + case <-finalTimer: + t.Fatal("did not receive a snapshot after the token manifested") + } + } + +} + +func testWriteToken(t *testing.T, a *TestAgent, tok *api.ACLToken) string { + req, _ := http.NewRequest("PUT", "/v1/acl/token", jsonReader(tok)) + req.Header.Add("X-Consul-Token", "root") + resp := httptest.NewRecorder() + a.srv.h.ServeHTTP(resp, req) + require.Equal(t, http.StatusOK, resp.Code) + + dec := json.NewDecoder(resp.Body) + aclResp := &structs.ACLToken{} + require.NoError(t, dec.Decode(aclResp)) + return aclResp.SecretID +} diff --git a/agent/testagent.go b/agent/testagent.go index 54db5c72ba42..76d82a2f84d2 100644 --- a/agent/testagent.go +++ b/agent/testagent.go @@ -214,6 +214,9 @@ func (a *TestAgent) Start(t *testing.T) error { // Lower the maximum backoff period of a cache refresh just for // tests see #14956 for more. result.RuntimeConfig.Cache.CacheRefreshMaxWait = 1 * time.Second + + // Lower the resync interval for tests. + result.RuntimeConfig.LocalProxyConfigResyncInterval = 250 * time.Millisecond } return result, err } From 2f8de3c3d1f93137dce2f7fc1162b737454f7c3f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 16:08:57 -0600 Subject: [PATCH 083/421] Backport of NET-2903 Normalize weight for http routes into release/1.15.x (#16532) * backport of commit 10ef73a5a6e5c4e9161f6b3b27d27619ee621986 * backport of commit 99f5b726d2a228a56da61696917efacff869e341 * NET-2903 Normalize weight for http routes (#16512) * NET-2903 Normalize weight for http routes * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --------- Co-authored-by: Melisa Griffin Co-authored-by: Melisa Griffin Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --- .changelog/16512.txt | 3 +++ agent/structs/config_entry_routes.go | 4 +++- agent/structs/config_entry_routes_test.go | 22 +++++++++++++++++++ .../api-gateway/configuration/http-route.mdx | 3 ++- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 .changelog/16512.txt diff --git a/.changelog/16512.txt b/.changelog/16512.txt new file mode 100644 index 000000000000..288ff8aa45e7 --- /dev/null +++ b/.changelog/16512.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: fix HTTPRoute bug where service weights could be less than or equal to 0 and result in a downstream envoy protocol error +``` \ No newline at end of file diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 801e22f18cbe..1235723f89f2 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -100,7 +100,9 @@ func (e *HTTPRouteConfigEntry) Normalize() error { func normalizeHTTPService(service HTTPService) HTTPService { service.EnterpriseMeta.Normalize() - + if service.Weight <= 0 { + service.Weight = 1 + } return service } diff --git a/agent/structs/config_entry_routes_test.go b/agent/structs/config_entry_routes_test.go index 83ab8c4d79e0..37c20390a311 100644 --- a/agent/structs/config_entry_routes_test.go +++ b/agent/structs/config_entry_routes_test.go @@ -262,6 +262,28 @@ func TestHTTPRoute(t *testing.T) { }}, }, }, + "rule normalizes service weight": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-one", + Rules: []HTTPRouteRule{{ + Services: []HTTPService{ + { + Name: "test", + Weight: 0, + }, + { + Name: "test2", + Weight: -1, + }}, + }}, + }, + check: func(t *testing.T, entry ConfigEntry) { + route := entry.(*HTTPRouteConfigEntry) + require.Equal(t, 1, route.Rules[0].Services[0].Weight) + require.Equal(t, 1, route.Rules[0].Services[1].Weight) + }, + }, } testConfigEntryNormalizeAndValidate(t, cases) } diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx index c492e331e2ae..7ab1a506d462 100644 --- a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx +++ b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx @@ -630,7 +630,8 @@ Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partiti ### `Rules[].Services[].Weight` -Specifies the proportion of requests forwarded to the specified service. The +Specifies the proportion of requests forwarded to the specified service. If no weight is specified, or if the specified +weight is set to less than or equal to `0`, the weight is normalized to `1`. The proportion is determined by dividing the value of the weight by the sum of all weights in the service list. For non-zero values, there may be some deviation from the exact proportion depending on the precision an implementation From e69ed7a9564725e4952f690f7f0fdc00d1bf742b Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 16:24:18 -0600 Subject: [PATCH 084/421] Backport of Add some basic UI improvements for api-gateway services into release/1.15.x (#16533) * backport of commit 06046ae56f44ecd96ec6e3fb3aaf88d982104833 * backport of commit 884d72b392d113458931d9ea2f20c176daade176 * backport of commit 9ff6b8bb5c8faddff422b401f31db3eb4bf9ee3c * backport of commit 5a3e7ec13953d644086426c1869870edb0f40047 * backport of commit 38fec2d0d8c3359f281ab61ab29a232df71abfb1 * backport of commit 9844e073118bdbf4b905a67bfe384fc773e3c895 --------- Co-authored-by: Andrew Stucki --- .changelog/16508.txt | 3 + .../app/components/consul/kind/index.hbs | 182 +++++++++--------- .../app/components/consul/kind/index.js | 14 ++ .../consul/service/search-bar/index.hbs | 2 +- .../app/filter/predicates/service.js | 1 + .../consul-ui/app/models/service-instance.js | 10 +- .../mock-api/v1/internal/ui/exported-services | 2 +- .../mock-api/v1/internal/ui/services | 2 +- .../acceptance/dc/services/index.feature | 10 +- .../dc/services/show/services.feature | 1 + .../consul-ui/translations/common/en-us.yaml | 1 + 11 files changed, 134 insertions(+), 94 deletions(-) create mode 100644 .changelog/16508.txt diff --git a/.changelog/16508.txt b/.changelog/16508.txt new file mode 100644 index 000000000000..4732ed553c61 --- /dev/null +++ b/.changelog/16508.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: support filtering API gateways in the ui and displaying their documentation links +``` diff --git a/ui/packages/consul-ui/app/components/consul/kind/index.hbs b/ui/packages/consul-ui/app/components/consul/kind/index.hbs index e5bfc6d4423c..58b981545612 100644 --- a/ui/packages/consul-ui/app/components/consul/kind/index.hbs +++ b/ui/packages/consul-ui/app/components/consul/kind/index.hbs @@ -1,89 +1,99 @@ {{#if item.Kind}} - {{#let (titleize (humanize item.Kind)) as |Name|}} - {{#if withInfo}} -
    -
    - - {{Name}} - -
    -
    - - - {{#if (eq item.Kind 'ingress-gateway')}} - Ingress gateways enable ingress traffic from services outside the Consul service mesh to services inside the Consul service mesh. - {{else if (eq item.Kind 'terminating-gateway')}} - Terminating gateways allow connect-enabled services in Consul service mesh to communicate with services outside the service mesh. - {{else}} - Mesh gateways enable routing of Connect traffic between different Consul datacenters. - {{/if}} - - -
  • - {{#if (eq item.Kind 'ingress-gateway')}} - About Ingress gateways - {{else if (eq item.Kind 'terminating-gateway')}} - About Terminating gateways - {{else}} - About Mesh gateways - {{/if}} -
    • - {{#let (from-entries (array - (array 'ingress-gateway' '/consul/developer-mesh/ingress-gateways') - (array 'terminating-gateway' '/consul/developer-mesh/understand-terminating-gateways') - (array 'mesh-gateway' '/consul/developer-mesh/connect-gateways') - ) - ) as |link|}} -
  • - - Learn guides - -
    • - {{/let}} - {{#let (from-entries (array - (array 'ingress-gateway' '/connect/ingress-gateway') - (array 'terminating-gateway' '/connect/terminating-gateway') - (array 'mesh-gateway' '/connect/mesh-gateway') - ) - ) as |link|}} -
  • - - Documentation - -
    • -
  • - Other gateway types -
    • - {{#if (not-eq item.Kind 'mesh-gateway')}} -
  • - - Mesh gateways - -
    • + {{#if withInfo}} +
    +
    + + {{Name}} + +
    +
    + + + {{#if (eq item.Kind 'ingress-gateway')}} + Ingress gateways enable ingress traffic from services outside the Consul service mesh to services inside the Consul service mesh. + {{else if (eq item.Kind 'terminating-gateway')}} + Terminating gateways allow connect-enabled services in Consul service mesh to communicate with services outside the service mesh. + {{else if (eq item.Kind 'api-gateway')}} + API gateways enable ingress traffic from services outside the Consul service mesh to services inside the Consul service mesh. + {{else}} + Mesh gateways enable routing of Connect traffic between different Consul datacenters. + {{/if}} + + +
  • + {{#if (eq item.Kind 'ingress-gateway')}} + About Ingress gateways + {{else if (eq item.Kind 'terminating-gateway')}} + About Terminating gateways + {{else if (eq item.Kind 'api-gateway')}} + About API gateways + {{else}} + About Mesh gateways {{/if}} - {{#if (not-eq item.Kind 'terminating-gateway')}} -
  • - - Terminating gateways - -
    • - {{/if}} - {{#if (not-eq item.Kind 'ingress-gateway')}} -
  • - - Ingress gateways - -
    • - {{/if}} - {{/let}} -     -     -
    -
    - {{else}} - - {{Name}} - - {{/if}} - {{/let}} + + {{#let (from-entries (array + (array 'ingress-gateway' '/consul/developer-mesh/ingress-gateways') + (array 'terminating-gateway' '/consul/developer-mesh/understand-terminating-gateways') + (array 'mesh-gateway' '/consul/developer-mesh/connect-gateways') + ) + ) as |link|}} +
  • + + Learn guides + +
    • + {{/let}} + {{#let (from-entries (array + (array 'ingress-gateway' '/connect/gateways/ingress-gateway') + (array 'terminating-gateway' '/connect/gateways/terminating-gateway') + (array 'api-gateway' '/connect/gateways/api-gateway') + (array 'mesh-gateway' '/connect/gateways/mesh-gateway') + ) + ) as |link|}} +
  • + + Documentation + +
    • +
  • + Other gateway types +
    • + {{#if (not-eq item.Kind 'mesh-gateway')}} +
  • + + Mesh gateways + +
    • + {{/if}} + {{#if (not-eq item.Kind 'terminating-gateway')}} +
  • + + Terminating gateways + +
    • + {{/if}} + {{#if (not-eq item.Kind 'ingress-gateway')}} +
  • + + Ingress gateways + +
    • + {{/if}} + {{#if (not-eq item.Kind 'api-gateway')}} +
  • + + API gateways + +
    • + {{/if}} + {{/let}} + + +
    +
    + {{else}} + + {{Name}} + + {{/if}} {{/if}} diff --git a/ui/packages/consul-ui/app/components/consul/kind/index.js b/ui/packages/consul-ui/app/components/consul/kind/index.js index 4798652642ba..63117c19fe41 100644 --- a/ui/packages/consul-ui/app/components/consul/kind/index.js +++ b/ui/packages/consul-ui/app/components/consul/kind/index.js @@ -1,5 +1,19 @@ import Component from '@ember/component'; +import { computed } from '@ember/object'; +import { titleize } from 'ember-cli-string-helpers/helpers/titleize'; +import { humanize } from 'ember-cli-string-helpers/helpers/humanize'; + +const normalizedGatewayLabels = { + 'api-gateway': 'API Gateway', + 'mesh-gateway': 'Mesh Gateway', + 'ingress-gateway': 'Ingress Gateway', + 'terminating-gateway': 'Terminating Gateway', +}; export default Component.extend({ tagName: '', + Name: computed('item.Kind', function () { + const name = normalizedGatewayLabels[this.item.Kind]; + return name ? name : titleize(humanize(this.item.Kind)); + }), }); diff --git a/ui/packages/consul-ui/app/components/consul/service/search-bar/index.hbs b/ui/packages/consul-ui/app/components/consul/service/search-bar/index.hbs index 5c76cf501ca2..bda5097c9c2b 100644 --- a/ui/packages/consul-ui/app/components/consul/service/search-bar/index.hbs +++ b/ui/packages/consul-ui/app/components/consul/service/search-bar/index.hbs @@ -102,7 +102,7 @@ {{t 'common.consul.service'}} - {{#each (array 'ingress-gateway' 'terminating-gateway' 'mesh-gateway') as |kind|}} + {{#each (array 'api-gateway' 'ingress-gateway' 'terminating-gateway' 'mesh-gateway') as |kind|}} diff --git a/ui/packages/consul-ui/app/filter/predicates/service.js b/ui/packages/consul-ui/app/filter/predicates/service.js index 14bae4384675..a5030e34ed25 100644 --- a/ui/packages/consul-ui/app/filter/predicates/service.js +++ b/ui/packages/consul-ui/app/filter/predicates/service.js @@ -2,6 +2,7 @@ import setHelpers from 'mnemonist/set'; export default { kind: { + 'api-gateway': (item, value) => item.Kind === value, 'ingress-gateway': (item, value) => item.Kind === value, 'terminating-gateway': (item, value) => item.Kind === value, 'mesh-gateway': (item, value) => item.Kind === value, diff --git a/ui/packages/consul-ui/app/models/service-instance.js b/ui/packages/consul-ui/app/models/service-instance.js index 1863dd6bbabc..51f98e25b53d 100644 --- a/ui/packages/consul-ui/app/models/service-instance.js +++ b/ui/packages/consul-ui/app/models/service-instance.js @@ -64,9 +64,13 @@ export default class ServiceInstance extends Model { @computed('Service.Kind') get IsProxy() { - return ['connect-proxy', 'mesh-gateway', 'ingress-gateway', 'terminating-gateway'].includes( - this.Service.Kind - ); + return [ + 'connect-proxy', + 'mesh-gateway', + 'ingress-gateway', + 'terminating-gateway', + 'api-gateway', + ].includes(this.Service.Kind); } // IsOrigin means that the service can have associated up or down streams, diff --git a/ui/packages/consul-ui/mock-api/v1/internal/ui/exported-services b/ui/packages/consul-ui/mock-api/v1/internal/ui/exported-services index 8bfe712541b4..f138a28d9f63 100644 --- a/ui/packages/consul-ui/mock-api/v1/internal/ui/exported-services +++ b/ui/packages/consul-ui/mock-api/v1/internal/ui/exported-services @@ -3,7 +3,7 @@ ${[0].map( () => { let prevKind; let name; - const gateways = ['mesh-gateway', 'ingress-gateway', 'terminating-gateway']; + const gateways = ['mesh-gateway', 'ingress-gateway', 'terminating-gateway', 'api-gateway']; return ` [ ${ diff --git a/ui/packages/consul-ui/mock-api/v1/internal/ui/services b/ui/packages/consul-ui/mock-api/v1/internal/ui/services index f44e59d179c5..e29f7feefb7c 100644 --- a/ui/packages/consul-ui/mock-api/v1/internal/ui/services +++ b/ui/packages/consul-ui/mock-api/v1/internal/ui/services @@ -2,7 +2,7 @@ ${[0].map( () => { let prevKind; let name; - const gateways = ['mesh-gateway', 'ingress-gateway', 'terminating-gateway']; + const gateways = ['mesh-gateway', 'ingress-gateway', 'terminating-gateway', 'api-gateway']; return ` [ ${ diff --git a/ui/packages/consul-ui/tests/acceptance/dc/services/index.feature b/ui/packages/consul-ui/tests/acceptance/dc/services/index.feature index ecfbc803230a..df0ebc2dde39 100644 --- a/ui/packages/consul-ui/tests/acceptance/dc/services/index.feature +++ b/ui/packages/consul-ui/tests/acceptance/dc/services/index.feature @@ -71,7 +71,7 @@ Feature: dc / services / index: List Services --- Scenario: Viewing the service list page with gateways Given 1 datacenter model with the value "dc-1" - And 3 service models from yaml + And 4 service models from yaml --- - Name: Service-0-proxy Kind: 'connect-proxy' @@ -88,6 +88,11 @@ Feature: dc / services / index: List Services ChecksPassing: 0 ChecksWarning: 0 ChecksCritical: 1 + - Name: Service-3-api-gateway + Kind: 'api-gateway' + ChecksPassing: 0 + ChecksWarning: 0 + ChecksCritical: 1 --- When I visit the services page for yaml @@ -96,11 +101,12 @@ Feature: dc / services / index: List Services --- Then the url should be /dc-1/services And the title should be "Services - Consul" - Then I see 2 service models + Then I see 3 service models And I see kind on the services like yaml --- - ingress-gateway - terminating-gateway + - api-gateway --- Scenario: View a Service in mesh Given 1 datacenter model with the value "dc-1" diff --git a/ui/packages/consul-ui/tests/acceptance/dc/services/show/services.feature b/ui/packages/consul-ui/tests/acceptance/dc/services/show/services.feature index 3b19a6995934..b6819558d845 100644 --- a/ui/packages/consul-ui/tests/acceptance/dc/services/show/services.feature +++ b/ui/packages/consul-ui/tests/acceptance/dc/services/show/services.feature @@ -50,6 +50,7 @@ Feature: dc / services / show / services | Name | Kind | | service | ~ | | ingress-gateway | ingress-gateway | + | api-gateway | api-gateway | | mesh-gateway | mesh-gateway | --------------------------------------------- diff --git a/ui/packages/consul-ui/translations/common/en-us.yaml b/ui/packages/consul-ui/translations/common/en-us.yaml index 5c584540eda6..18180f2073a1 100644 --- a/ui/packages/consul-ui/translations/common/en-us.yaml +++ b/ui/packages/consul-ui/translations/common/en-us.yaml @@ -32,6 +32,7 @@ consul: ingress-gateway: Ingress Gateway terminating-gateway: Terminating Gateway mesh-gateway: Mesh Gateway + api-gateway: API Gateway status: Health Status service.meta: Service Meta node.meta: Node Meta From a95d028b0665bfb2a26b9d806a63b1e55523bc3e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 3 Mar 2023 17:04:58 -0600 Subject: [PATCH 085/421] backport of commit 5dca39b8137daf740f18b6d86f1d48081c317815 (#16536) Co-authored-by: trujillo-adam --- website/content/docs/services/discovery/dns-static-lookups.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/services/discovery/dns-static-lookups.mdx b/website/content/docs/services/discovery/dns-static-lookups.mdx index 68191104a2a4..aa2524ec4574 100644 --- a/website/content/docs/services/discovery/dns-static-lookups.mdx +++ b/website/content/docs/services/discovery/dns-static-lookups.mdx @@ -39,7 +39,7 @@ Specify the name of the node, datacenter, and domain using the following FQDN sy The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of the agent. -By default, the domain is `consul`. Refer to [Configure DNS Behaviors]() for information about using alternate domains. +By default, the domain is `consul`. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for information about using alternate domains. ### Node lookup results From 88496d7f8a2686ac709d8eb4387fed1165ab377f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 6 Mar 2023 09:11:27 -0600 Subject: [PATCH 086/421] NET-2904 Fixes API Gateway Route Service Weight Division Error (#16540) Co-authored-by: Melisa Griffin --- .changelog/16531.txt | 3 +++ agent/xds/routes.go | 14 ++++++++++++-- .../envoy/case-api-gateway-http-simple/setup.sh | 4 ++++ .../envoy/case-api-gateway-http-simple/verify.bats | 4 ++-- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 .changelog/16531.txt diff --git a/.changelog/16531.txt b/.changelog/16531.txt new file mode 100644 index 000000000000..71f83ad2acc2 --- /dev/null +++ b/.changelog/16531.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: fix HTTPRoute bug where services with a weight not divisible by 10000 are never registered properly +``` \ No newline at end of file diff --git a/agent/xds/routes.go b/agent/xds/routes.go index e80c3c2476e2..9a02d96ea5c8 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -863,6 +863,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( forMeshGateway bool, ) (*envoy_route_v3.Route_Route, error) { clusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(splits)) + totalWeight := 0 for _, split := range splits { nextNode := chain.Nodes[split.NextNode] @@ -878,8 +879,10 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( // The smallest representable weight is 1/10000 or .01% but envoy // deals with integers so scale everything up by 100x. + weight := int(split.Weight * 100) + totalWeight += weight cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ - Weight: makeUint32Value(int(split.Weight * 100)), + Weight: makeUint32Value(weight), Name: targetOptions.clusterName, } if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { @@ -893,12 +896,19 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( return nil, fmt.Errorf("number of clusters in splitter must be > 0; got %d", len(clusters)) } + envoyWeightScale := 10000 + if envoyWeightScale < totalWeight { + clusters[0].Weight.Value += uint32(totalWeight - envoyWeightScale) + } else { + clusters[0].Weight.Value += uint32(envoyWeightScale - totalWeight) + } + return &envoy_route_v3.Route_Route{ Route: &envoy_route_v3.RouteAction{ ClusterSpecifier: &envoy_route_v3.RouteAction_WeightedClusters{ WeightedClusters: &envoy_route_v3.WeightedCluster{ Clusters: clusters, - TotalWeight: makeUint32Value(10000), // scaled up 100% + TotalWeight: makeUint32Value(envoyWeightScale), // scaled up 100% }, }, }, diff --git a/test/integration/connect/envoy/case-api-gateway-http-simple/setup.sh b/test/integration/connect/envoy/case-api-gateway-http-simple/setup.sh index 8d0513553de7..6dab478e203b 100644 --- a/test/integration/connect/envoy/case-api-gateway-http-simple/setup.sh +++ b/test/integration/connect/envoy/case-api-gateway-http-simple/setup.sh @@ -35,6 +35,10 @@ rules = [ services = [ { name = "s1" + }, + { + name = "s2" + weight = 2 } ] } diff --git a/test/integration/connect/envoy/case-api-gateway-http-simple/verify.bats b/test/integration/connect/envoy/case-api-gateway-http-simple/verify.bats index c7378e55bfef..72686b3c4f25 100644 --- a/test/integration/connect/envoy/case-api-gateway-http-simple/verify.bats +++ b/test/integration/connect/envoy/case-api-gateway-http-simple/verify.bats @@ -22,9 +22,9 @@ load helpers } @test "api gateway should be able to connect to s1 via configured port" { - run retry_long curl -s -f -d hello localhost:9999 + run retry_long curl -s -d hello localhost:9999 [ "$status" -eq 0 ] - [[ "$output" == *"hello"* ]] + [[ ! -z "$output" ]] } @test "api gateway should get an intentions error connecting to s2 via configured port" { From 583466b6d5d9922ffcb6240a23c1fd1d29156773 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 6 Mar 2023 09:20:49 -0600 Subject: [PATCH 087/421] Backport of Improve ux around ACL token to help users avoid overwriting node/service identities into release/1.15.x (#16541) * backport of commit 04a7185e76a258f8b79c6f6b427d0368f38e5076 * backport of commit f47fbf7c74f14d6c85e47b931d079016b6af47a0 * backport of commit bf9fb378684986fdef42edc375fd85a996b3ec25 * backport of commit 22fde7628e00d9ef9a6cedbf61a4843d5c9e4a6d * backport of commit 0313fa653ae5f109ca7148eaa3ebc6633ae7e8f2 * backport of commit 6e19413a84224ab737a19c044d2182feba5242ce * backport of commit e1fb12f0730afd7a71f8cf5f9815cce5f3f5e78c * backport of commit 4beecd136e50bc6267db3db71e7bab90754ad2b7 --------- Co-authored-by: Ronald Ekambi --- .changelog/16506.txt | 8 ++ command/acl/token/update/token_update.go | 83 +++++++++++++------ command/acl/token/update/token_update_test.go | 43 ++++++++++ .../content/commands/acl/policy/update.mdx | 2 + website/content/commands/acl/token/update.mdx | 18 +++- 5 files changed, 125 insertions(+), 29 deletions(-) create mode 100644 .changelog/16506.txt diff --git a/.changelog/16506.txt b/.changelog/16506.txt new file mode 100644 index 000000000000..2560c2474665 --- /dev/null +++ b/.changelog/16506.txt @@ -0,0 +1,8 @@ +```release-note:deprecation +cli: Deprecate the `-merge-node-identites` and `-merge-service-identities` flags from the `consul token update` command in favor of: `-append-node-identity` and `-append-service-identity`. +``` + +```release-note:improvement +cli: added `-append-service-identity` and `-append-node-identity` flags to the `consul token update` command. +These flags allow updates to a token's node identities/service identities without having to override them. +``` \ No newline at end of file diff --git a/command/acl/token/update/token_update.go b/command/acl/token/update/token_update.go index 6cb529edd45f..519a3da8d4e4 100644 --- a/command/acl/token/update/token_update.go +++ b/command/acl/token/update/token_update.go @@ -25,37 +25,35 @@ type cmd struct { http *flags.HTTPFlags help string - tokenAccessorID string - policyIDs []string - appendPolicyIDs []string - policyNames []string - appendPolicyNames []string - roleIDs []string - appendRoleIDs []string - roleNames []string - appendRoleNames []string - serviceIdents []string - nodeIdents []string - description string - mergeServiceIdents bool - mergeNodeIdents bool - showMeta bool - format string + tokenAccessorID string + policyIDs []string + appendPolicyIDs []string + policyNames []string + appendPolicyNames []string + roleIDs []string + appendRoleIDs []string + roleNames []string + appendRoleNames []string + serviceIdents []string + nodeIdents []string + appendNodeIdents []string + appendServiceIdents []string + description string + showMeta bool + format string // DEPRECATED - mergeRoles bool - mergePolicies bool - tokenID string + mergeServiceIdents bool + mergeNodeIdents bool + mergeRoles bool + mergePolicies bool + tokenID string } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.showMeta, "meta", false, "Indicates that token metadata such "+ "as the content hash and raft indices should be shown for each entry") - c.flags.BoolVar(&c.mergeServiceIdents, "merge-service-identities", false, "Merge the new service identities "+ - "with the existing service identities") - c.flags.BoolVar(&c.mergeNodeIdents, "merge-node-identities", false, "Merge the new node identities "+ - "with the existing node identities") c.flags.StringVar(&c.tokenAccessorID, "accessor-id", "", "The Accessor ID of the token to update. "+ "It may be specified as a unique ID prefix but will error if the prefix "+ "matches multiple token Accessor IDs") @@ -79,9 +77,15 @@ func (c *cmd) init() { c.flags.Var((*flags.AppendSliceValue)(&c.serviceIdents), "service-identity", "Name of a "+ "service identity to use for this token. May be specified multiple times. Format is "+ "the SERVICENAME or SERVICENAME:DATACENTER1,DATACENTER2,...") + c.flags.Var((*flags.AppendSliceValue)(&c.appendServiceIdents), "append-service-identity", "Name of a "+ + "service identity to use for this token. This token retains existing service identities. May be specified"+ + "multiple times. Format is the SERVICENAME or SERVICENAME:DATACENTER1,DATACENTER2,...") c.flags.Var((*flags.AppendSliceValue)(&c.nodeIdents), "node-identity", "Name of a "+ "node identity to use for this token. May be specified multiple times. Format is "+ "NODENAME:DATACENTER") + c.flags.Var((*flags.AppendSliceValue)(&c.appendNodeIdents), "append-node-identity", "Name of a "+ + "node identity to use for this token. This token retains existing node identities. May be "+ + "specified multiple times. Format is NODENAME:DATACENTER") c.flags.StringVar( &c.format, "format", @@ -101,6 +105,10 @@ func (c *cmd) init() { "Use -append-policy-id or -append-policy-name instead.") c.flags.BoolVar(&c.mergeRoles, "merge-roles", false, "DEPRECATED. "+ "Use -append-role-id or -append-role-name instead.") + c.flags.BoolVar(&c.mergeServiceIdents, "merge-service-identities", false, "DEPRECATED. "+ + "Use -append-service-identity instead.") + c.flags.BoolVar(&c.mergeNodeIdents, "merge-node-identities", false, "DEPRECATED. "+ + "Use -append-node-identity instead.") } func (c *cmd) Run(args []string) int { @@ -147,13 +155,38 @@ func (c *cmd) Run(args []string) int { t.Description = c.description } + hasAppendServiceFields := len(c.appendServiceIdents) > 0 + hasServiceFields := len(c.serviceIdents) > 0 + if hasAppendServiceFields && hasServiceFields { + c.UI.Error("Cannot combine the use of service-identity flag with append-service-identity. " + + "To set or overwrite existing service identities, use -service-identity. " + + "To append to existing service identities, use -append-service-identity.") + return 1 + } + parsedServiceIdents, err := acl.ExtractServiceIdentities(c.serviceIdents) + if hasAppendServiceFields { + parsedServiceIdents, err = acl.ExtractServiceIdentities(c.appendServiceIdents) + } if err != nil { c.UI.Error(err.Error()) return 1 } + hasAppendNodeFields := len(c.appendNodeIdents) > 0 + hasNodeFields := len(c.nodeIdents) > 0 + + if hasAppendNodeFields && hasNodeFields { + c.UI.Error("Cannot combine the use of node-identity flag with append-node-identity. " + + "To set or overwrite existing node identities, use -node-identity. " + + "To append to existing node identities, use -append-node-identity.") + return 1 + } + parsedNodeIdents, err := acl.ExtractNodeIdentities(c.nodeIdents) + if hasAppendNodeFields { + parsedNodeIdents, err = acl.ExtractNodeIdentities(c.appendNodeIdents) + } if err != nil { c.UI.Error(err.Error()) return 1 @@ -310,7 +343,7 @@ func (c *cmd) Run(args []string) int { } } - if c.mergeServiceIdents { + if c.mergeServiceIdents || hasAppendServiceFields { for _, svcid := range parsedServiceIdents { found := -1 for i, link := range t.ServiceIdentities { @@ -330,7 +363,7 @@ func (c *cmd) Run(args []string) int { t.ServiceIdentities = parsedServiceIdents } - if c.mergeNodeIdents { + if c.mergeNodeIdents || hasAppendNodeFields { for _, nodeid := range parsedNodeIdents { found := false for _, link := range t.NodeIdentities { diff --git a/command/acl/token/update/token_update_test.go b/command/acl/token/update/token_update_test.go index bd11d1d8e3f6..815702841044 100644 --- a/command/acl/token/update/token_update_test.go +++ b/command/acl/token/update/token_update_test.go @@ -109,6 +109,22 @@ func TestTokenUpdateCommand(t *testing.T) { require.ElementsMatch(t, expected, token.NodeIdentities) }) + // update with append-node-identity + t.Run("append-node-identity", func(t *testing.T) { + + token := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-node-identity=third:node", + "-description=test token", + }) + + require.Len(t, token.NodeIdentities, 3) + require.Equal(t, "third", token.NodeIdentities[2].NodeName) + require.Equal(t, "node", token.NodeIdentities[2].Datacenter) + }) + // update with policy by name t.Run("policy-name", func(t *testing.T) { token := run(t, []string{ @@ -135,6 +151,33 @@ func TestTokenUpdateCommand(t *testing.T) { require.Len(t, token.Policies, 1) }) + // update with service-identity + t.Run("service-identity", func(t *testing.T) { + token := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-service-identity=service:datapalace", + "-description=test token", + }) + + require.Len(t, token.ServiceIdentities, 1) + require.Equal(t, "service", token.ServiceIdentities[0].ServiceName) + }) + + // update with append-service-identity + t.Run("append-service-identity", func(t *testing.T) { + token := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-service-identity=web", + "-description=test token", + }) + require.Len(t, token.ServiceIdentities, 2) + require.Equal(t, "web", token.ServiceIdentities[1].ServiceName) + }) + // update with no description shouldn't delete the current description t.Run("merge-description", func(t *testing.T) { token := run(t, []string{ diff --git a/website/content/commands/acl/policy/update.mdx b/website/content/commands/acl/policy/update.mdx index e62dfa72d997..f64a1f79068e 100644 --- a/website/content/commands/acl/policy/update.mdx +++ b/website/content/commands/acl/policy/update.mdx @@ -49,6 +49,8 @@ Usage: `consul acl policy update [options] [args]` the value is a file path to load the rules from. `-` may also be given to indicate that the rules are available on stdin. +~> Specifying `-rules` will overwrite existing rules. + - `-valid-datacenter=` - Datacenter that the policy should be valid within. This flag may be specified multiple times. diff --git a/website/content/commands/acl/token/update.mdx b/website/content/commands/acl/token/update.mdx index 19441e1020b0..1a1703cb143b 100644 --- a/website/content/commands/acl/token/update.mdx +++ b/website/content/commands/acl/token/update.mdx @@ -33,8 +33,9 @@ Usage: `consul acl token update [options]` - `-id=` - The Accessor ID of the token to read. It may be specified as a unique ID prefix but will error if the prefix matches multiple token Accessor IDs -- `merge-node-identities` - Merge the new node identities with the existing node +- `-merge-node-identities` - Deprecated. Merge the new node identities with the existing node identities. +~> This is deprecated and will be removed in a future Consul version. Use `append-node-identity` instead. - `-merge-policies` - Deprecated. Merge the new policies with the existing policies. @@ -46,15 +47,20 @@ instead. ~> This is deprecated and will be removed in a future Consul version. Use `append-role-id` or `append-role-name` instead. -- `-merge-service-identities` - Merge the new service identities with the existing service identities. +- `-merge-service-identities` - Deprecated. Merge the new service identities with the existing service identities. + +~> This is deprecated and will be removed in a future Consul version. Use `append-service-identity` instead. - `-meta` - Indicates that token metadata such as the content hash and Raft indices should be shown for each entry. -- `-node-identity=` - Name of a node identity to use for this role. May +- `-node-identity=` - Name of a node identity to use for this role. Overwrites existing node identity. May be specified multiple times. Format is `NODENAME:DATACENTER`. Added in Consul 1.8.1. +- `-append-node-identity=` - Name of a node identity to add to this role. May + be specified multiple times. The token retains existing node identities. Format is `NODENAME:DATACENTER`. + - `-policy-id=` - ID of a policy to use for this token. Overwrites existing policies. May be specified multiple times. - `-policy-name=` - Name of a policy to use for this token. Overwrites existing policies. May be specified multiple times. @@ -76,9 +82,13 @@ instead. - `-append-role-name=` - Name of a role to add to this token. The token retains existing roles. May be specified multiple times. - `-service-identity=` - Name of a service identity to use for this - token. May be specified multiple times. Format is the `SERVICENAME` or + token. Overwrites existing service identities. May be specified multiple times. Format is the `SERVICENAME` or `SERVICENAME:DATACENTER1,DATACENTER2,...` +- `-append-service-identity=` - Name of a service identity to add to this + token. May be specified multiple times. The token retains existing service identities. + Format is the `SERVICENAME` or `SERVICENAME:DATACENTER1,DATACENTER2,...` + - `-format={pretty|json}` - Command output format. The default value is `pretty`. #### Enterprise Options From 494fba26ec4745958110aa44aee50579af7e2669 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 6 Mar 2023 10:22:44 -0600 Subject: [PATCH 088/421] Backport of Follow-up fixes to consul connect envoy command into release/1.15.x (#16542) * backport of commit dc7f05b4e829a763b72ec1faf7da9b83cf5fee1f * backport of commit d0adb5c45c95bffe66a4980f388729bbd900d508 * backport of commit 097143b273e46ebb48780710e3b621d0ed7466d9 * backport of commit db403debaf2a1d573fceff0cacda5da2cdfccc8a --------- Co-authored-by: Chris S. Kim --- .changelog/16530.txt | 7 +++++++ command/connect/envoy/envoy.go | 31 +++++++++++++++++++---------- command/connect/envoy/envoy_test.go | 28 ++++++++++++++++---------- 3 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 .changelog/16530.txt diff --git a/.changelog/16530.txt b/.changelog/16530.txt new file mode 100644 index 000000000000..38d98036ab9b --- /dev/null +++ b/.changelog/16530.txt @@ -0,0 +1,7 @@ +```release-note:bug +cli: Fixes an issue with `consul connect envoy` where a log to STDOUT could malform JSON when used with `-bootstrap`. +``` + +```release-note:bug +cli: Fixes an issue with `consul connect envoy` where grpc-disabled agents were not error-handled correctly. +``` diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index 102c16494db3..0864ecc19973 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/hashicorp/consul/api" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" "github.com/mitchellh/cli" @@ -22,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds" "github.com/hashicorp/consul/agent/xds/accesslogs" + "github.com/hashicorp/consul/api" proxyCmd "github.com/hashicorp/consul/command/connect/proxy" "github.com/hashicorp/consul/command/flags" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -810,8 +810,7 @@ func (c *cmd) xdsAddress() (GRPC, error) { port, protocol, err := c.lookupXDSPort() if err != nil { if strings.Contains(err.Error(), "Permission denied") { - // Token did not have agent:read. Log and proceed with defaults. - c.UI.Info(fmt.Sprintf("Could not query /v1/agent/self for xDS ports: %s", err)) + // Token did not have agent:read. Suppress and proceed with defaults. } else { // If not a permission denied error, gRPC is explicitly disabled // or something went fatally wrong. @@ -822,7 +821,7 @@ func (c *cmd) xdsAddress() (GRPC, error) { // This is the dev mode default and recommended production setting if // enabled. port = 8502 - c.UI.Info("-grpc-addr not provided and unable to discover a gRPC address for xDS. Defaulting to localhost:8502") + c.UI.Warn("-grpc-addr not provided and unable to discover a gRPC address for xDS. Defaulting to localhost:8502") } addr = fmt.Sprintf("%vlocalhost:%v", protocol, port) } @@ -887,9 +886,12 @@ func (c *cmd) lookupXDSPort() (int, string, error) { var resp response if err := mapstructure.Decode(self, &resp); err == nil { - if resp.XDS.Ports.TLS < 0 && resp.XDS.Ports.Plaintext < 0 { - return 0, "", fmt.Errorf("agent has grpc disabled") - } + // When we get rid of the 1.10 compatibility code below we can uncomment + // this check: + // + // if resp.XDS.Ports.TLS <= 0 && resp.XDS.Ports.Plaintext <= 0 { + // return 0, "", fmt.Errorf("agent has grpc disabled") + // } if resp.XDS.Ports.TLS > 0 { return resp.XDS.Ports.TLS, "https://", nil } @@ -898,9 +900,12 @@ func (c *cmd) lookupXDSPort() (int, string, error) { } } - // If above TLS and Plaintext ports are both 0, fallback to - // old API for the case where a new consul CLI is being used - // with an older API version. + // If above TLS and Plaintext ports are both 0, it could mean + // gRPC is disabled on the agent or we are using an older API. + // In either case, fallback to reading from the DebugConfig. + // + // Next major version we should get rid of this below code. + // It exists for compatibility reasons for 1.10 and below. cfg, ok := self["DebugConfig"] if !ok { return 0, "", fmt.Errorf("unexpected agent response: no debug config") @@ -914,6 +919,12 @@ func (c *cmd) lookupXDSPort() (int, string, error) { return 0, "", fmt.Errorf("invalid grpc port in agent response") } + // This works for both <1.10 and later but we should prefer + // reading from resp.XDS instead. + if portN < 0 { + return 0, "", fmt.Errorf("agent has grpc disabled") + } + return int(portN), "", nil } diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 09c02c91c094..e2692c77d7c3 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -13,7 +13,6 @@ import ( "strings" "testing" - "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/mitchellh/cli" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent" "github.com/hashicorp/consul/agent/xds" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" ) @@ -123,6 +123,7 @@ type generateConfigTestCase struct { NamespacesEnabled bool XDSPorts agent.GRPCPorts // used to mock an agent's configured gRPC ports. Plaintext defaults to 8502 and TLS defaults to 8503. AgentSelf110 bool // fake the agent API from versions v1.10 and earlier + GRPCDisabled bool WantArgs BootstrapTplArgs WantErr string WantWarn string @@ -146,13 +147,10 @@ func TestGenerateConfig(t *testing.T) { WantErr: "'-node-name' requires '-proxy-id'", }, { - Name: "gRPC disabled", - Flags: []string{"-proxy-id", "test-proxy"}, - XDSPorts: agent.GRPCPorts{ - Plaintext: -1, - TLS: -1, - }, - WantErr: "agent has grpc disabled", + Name: "gRPC disabled", + Flags: []string{"-proxy-id", "test-proxy"}, + GRPCDisabled: true, + WantErr: "agent has grpc disabled", }, { Name: "defaults", @@ -1387,7 +1385,7 @@ func testMockAgent(tc generateConfigTestCase) http.HandlerFunc { case strings.Contains(r.URL.Path, "/agent/service"): testMockAgentProxyConfig(tc.ProxyConfig, tc.NamespacesEnabled)(w, r) case strings.Contains(r.URL.Path, "/agent/self"): - testMockAgentSelf(tc.XDSPorts, tc.AgentSelf110)(w, r) + testMockAgentSelf(tc.XDSPorts, tc.AgentSelf110, tc.GRPCDisabled)(w, r) case strings.Contains(r.URL.Path, "/catalog/node-services"): testMockCatalogNodeServiceList()(w, r) case strings.Contains(r.URL.Path, "/config/proxy-defaults/global"): @@ -1658,7 +1656,11 @@ func TestEnvoyCommand_canBindInternal(t *testing.T) { // testMockAgentSelf returns an empty /v1/agent/self response except GRPC // port is filled in to match the given wantXDSPort argument. -func testMockAgentSelf(wantXDSPorts agent.GRPCPorts, agentSelf110 bool) http.HandlerFunc { +func testMockAgentSelf( + wantXDSPorts agent.GRPCPorts, + agentSelf110 bool, + grpcDisabled bool, +) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { resp := agent.Self{ Config: map[string]interface{}{ @@ -1670,6 +1672,12 @@ func testMockAgentSelf(wantXDSPorts agent.GRPCPorts, agentSelf110 bool) http.Han resp.DebugConfig = map[string]interface{}{ "GRPCPort": wantXDSPorts.Plaintext, } + } else if grpcDisabled { + resp.DebugConfig = map[string]interface{}{ + "GRPCPort": -1, + } + // the real agent does not populate XDS if grpc or + // grpc-tls ports are < 0 } else { resp.XDS = &agent.XDSSelf{ // The deprecated Port field should default to TLS if it's available. From e66f26b30667eed9b311465d10aac274f54e4028 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 6 Mar 2023 15:54:48 -0600 Subject: [PATCH 089/421] Backport of Update the consul-k8s cli docs for the new `proxy log` subcommand into release/1.15.x (#16547) * backport of commit e877e0d09f7dd42dd8423eb36e3382c0d914a0c4 * backport of commit 853e8addf21783aea6edb2640b63bb091bbf90d9 * Update the consul-k8s cli docs for the new `proxy log` subcommand (#16458) * Update the consul-k8s cli docs for the new `proxy log` subcommand * Updated consul-k8s docs from PR feedback * Added proxy log command to release notes --------- Co-authored-by: jm96441n --- website/content/docs/k8s/k8s-cli.mdx | 286 ++++++++++++++++++ .../docs/release-notes/consul-k8s/v1_1_x.mdx | 23 +- 2 files changed, 300 insertions(+), 9 deletions(-) diff --git a/website/content/docs/k8s/k8s-cli.mdx b/website/content/docs/k8s/k8s-cli.mdx index b3e6f76bc4e7..bb45986e11e1 100644 --- a/website/content/docs/k8s/k8s-cli.mdx +++ b/website/content/docs/k8s/k8s-cli.mdx @@ -32,6 +32,7 @@ You can use the following commands with `consul-k8s`. - [`proxy`](#proxy): Inspect Envoy proxies managed by Consul. - [`proxy list`](#proxy-list): List all Pods running proxies managed by Consul. - [`proxy read`](#proxy-read): Inspect the Envoy configuration for a given Pod. + - [`proxy log`](#proxy-log): Inspect and modify the Envoy logging configuration for a given Pod. - [`status`](#status): Check the status of a Consul installation on Kubernetes. - [`troubleshoot`](#troubleshoot): Troubleshoot Consul service mesh and networking issues from a given pod. - [`uninstall`](#uninstall): Uninstall Consul deployment. @@ -101,6 +102,7 @@ Consul in your Kubernetes Cluster. - [`proxy list`](#proxy-list): List all Pods running proxies managed by Consul. - [`proxy read`](#proxy-read): Inspect the Envoy configuration for a given Pod. +- [`proxy log`](#proxy-log): Inspect and modify the Envoy logging configuration for a given Pod. ### `proxy list` @@ -448,6 +450,290 @@ $ consul-k8s proxy read backend-658b679b45-d5xlb -o raw } ``` +### `proxy log` + +The `proxy log` command allows you to inspect and modify the logging configuration of Envoy proxies running on a given Pod. + +```shell-session +$ consul-k8s proxy log +``` + +The command takes a required value, ``. This should be the full name +of a Kubernetes Pod. If a Pod is running more than one Envoy proxy managed by +Consul, as in the [Multiport configuration](/consul/docs/k8s/connect#kubernetes-pods-with-multiple-ports), +the terminal displays configuration information for all proxies in the pod. + +The following options are available. + +| Flag | Description | Default | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` Specifies the namespace containing the target Pod. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | +| `-update-level`, `-u` | `String` Specifies the logger (optional) and the level to update.

    Use the following format to configure the same level for loggers: `-update-level `.

    You can also specify a comma-delineated list to configure levels for specific loggers, for example: `-update-level grpc:warning,http:info`.

    | none | +| `-reset`, `-r` | `String` Reset the log levels for all loggers back to the default of `info` | `info` | + +#### Example commands +In the following example, Consul returns the log levels for all of an Envoy proxy's loggers in a pod with the ID `server-697458b9f8-4vr29`: + +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +rds info +backtrace info +hc info +http info +io info +jwt info +rocketmq info +matcher info +runtime info +redis info +stats info +tap info +alternate_protocols_cache info +grpc info +init info +quic info +thrift info +wasm info +aws info +conn_handler info +ext_proc info +hystrix info +tracing info +dns info +oauth2 info +connection info +health_checker info +kafka info +mongo info +config info +admin info +forward_proxy info +misc info +websocket info +dubbo info +happy_eyeballs info +main info +client info +lua info +udp info +cache_filter info +filter info +multi_connection info +quic_stream info +router info +http2 info +key_value_store info +secret info +testing info +upstream info +assert info +ext_authz info +rbac info +decompression info +envoy_bug info +file info +pool info +``` + +The following command updates the log levels for all loggers of an Envoy proxy to `warning`. +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -update-level warning +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +pool warning +rbac warning +tracing warning +aws warning +cache_filter warning +decompression warning +init warning +assert warning +client warning +misc warning +udp warning +config warning +hystrix warning +key_value_store warning +runtime warning +admin warning +dns warning +jwt warning +redis warning +quic warning +alternate_protocols_cache warning +conn_handler warning +ext_proc warning +http warning +oauth2 warning +ext_authz warning +http2 warning +kafka warning +mongo warning +router warning +thrift warning +grpc warning +matcher warning +hc warning +multi_connection warning +wasm warning +dubbo warning +filter warning +upstream warning +backtrace warning +connection warning +io warning +main warning +happy_eyeballs warning +rds warning +tap warning +envoy_bug warning +rocketmq warning +file warning +forward_proxy warning +stats warning +health_checker warning +lua warning +secret warning +quic_stream warning +testing warning +websocket warning +``` +The following command updates the `grpc` log level to `error`, the `http` log level to `critical`, and the `runtime` log level to `debug` for pod ID `server-697458b9f8-4vr29` +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -update-level grpc:error,http:critical,runtime:debug +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +assert info +dns info +http critical +pool info +thrift info +udp info +grpc error +hc info +stats info +wasm info +alternate_protocols_cache info +ext_authz info +filter info +http2 info +key_value_store info +tracing info +cache_filter info +quic_stream info +aws info +io info +matcher info +rbac info +tap info +connection info +conn_handler info +rocketmq info +hystrix info +oauth2 info +redis info +backtrace info +file info +forward_proxy info +kafka info +config info +router info +runtime debug +testing info +happy_eyeballs info +ext_proc info +init info +lua info +health_checker info +misc info +envoy_bug info +jwt info +main info +quic info +upstream info +websocket info +client info +decompression info +mongo info +multi_connection info +rds info +secret info +admin info +dubbo info +``` +The following command resets the log levels for all loggers of an Envoy proxy in pod `server-697458b9f8-4vr29` to the default level of `info`. +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -r +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +ext_proc info +secret info +thrift info +tracing info +dns info +rocketmq info +happy_eyeballs info +hc info +io info +misc info +conn_handler info +key_value_store info +rbac info +hystrix info +wasm info +admin info +cache_filter info +client info +health_checker info +oauth2 info +runtime info +testing info +grpc info +upstream info +forward_proxy info +matcher info +pool info +aws info +decompression info +jwt info +tap info +assert info +redis info +http info +quic info +rds info +connection info +envoy_bug info +stats info +alternate_protocols_cache info +backtrace info +filter info +http2 info +init info +multi_connection info +quic_stream info +dubbo info +ext_authz info +main info +udp info +websocket info +config info +mongo info +router info +file info +kafka info +lua info +``` ### `status` The `status` command provides an overall status summary of the Consul on Kubernetes installation. It also provides the configuration that was used to deploy Consul K8s and information about the health of Consul servers and clients. This command does not take in any flags. diff --git a/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx index 6931aecd7055..5c3b8a5619c8 100644 --- a/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx @@ -10,9 +10,14 @@ description: >- ## Release Highlights - **Enhanced Envoy Access Logging:** Envoy access logs are now centrally managed via the `accessLogs` field within the ProxyDefaults CRD to allow operators to easily turn on access logs for all proxies within the service mesh. Refer to [Access logs overview](/consul/docs/connect/observability/access-logs) for more information. - -- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. -The new `envoyExtensions` field in the ProxyDefaults and ServiceDefaults CRDs enable built-in Envoy extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions. + +- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. +The new `envoyExtensions` field in the ProxyDefaults and ServiceDefaults CRDs enable built-in Envoy extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions. + +- **Envoy Proxy Debugging CLI Commands**: This release adds a new command to quickly modify the log level of Envoy proxies for sidecars and gateways for easier debugging. +Refer to [consul-k8s CLI proxy log command](/consul/docs/k8s/k8s-cli#proxy-log) docs for more information. + * Add `consul-k8s proxy log podname` command for displaying current log levels or updating log levels for Envoy in a given pod. + ## What's Changed @@ -27,14 +32,14 @@ that you wanted to be injected, you must now set namespaceSelector as follows: operator: "NotIn" values: ["kube-system","local-path-storage"] ``` - + ## Supported Software ~> **Note:** Consul 1.14.x and 1.13.x are not supported. Please refer to [Supported Consul and Kubernetes versions](/consul/docs/k8s/compatibility#supported-consul-and-kubernetes-versions) for more detail on choosing the correct `consul-k8s` version. -- Consul 1.15.x. -- Consul Dataplane v1.1.x. Refer to [Envoy and Consul Dataplane](/consul/docs/connect/proxies/envoy#envoy-and-consul-dataplane) for details about Consul Dataplane versions and the available packaged Envoy version. +- Consul 1.15.x. +- Consul Dataplane v1.1.x. Refer to [Envoy and Consul Dataplane](/consul/docs/connect/proxies/envoy#envoy-and-consul-dataplane) for details about Consul Dataplane versions and the available packaged Envoy version. - Kubernetes 1.23.x - 1.26.x -- `kubectl` 1.23.x - 1.26.x +- `kubectl` 1.23.x - 1.26.x - Helm 3.6+ ## Upgrading @@ -43,9 +48,9 @@ For detailed information on upgrading, please refer to the [Upgrades page](/cons ## Known Issues -The following issues are known to exist in the v1.1.0 release: +The following issues are known to exist in the v1.1.0 release: -- Pod Security Standards that are configured for the [Pod Security Admission controller](https://kubernetes.io/blog/2022/08/25/pod-security-admission-stable/) are currently not supported by Consul K8s. OpenShift 4.11.x enables Pod Security Standards on Kubernetes 1.25 [by default](https://connect.redhat.com/en/blog/important-openshift-changes-pod-security-standards) and is also not supported. Support will be added in a future Consul K8s 1.0.x patch release. +- Pod Security Standards that are configured for the [Pod Security Admission controller](https://kubernetes.io/blog/2022/08/25/pod-security-admission-stable/) are currently not supported by Consul K8s. OpenShift 4.11.x enables Pod Security Standards on Kubernetes 1.25 [by default](https://connect.redhat.com/en/blog/important-openshift-changes-pod-security-standards) and is also not supported. Support will be added in a future Consul K8s 1.0.x patch release. ## Changelogs From dd740f0154204dbf46b30e9605efba52ca91e278 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 6 Mar 2023 17:36:51 -0600 Subject: [PATCH 090/421] Backport of Fix flakey tests related to ACL token updates into release/1.15.x (#16548) * backport of commit 76a5d2fac456bf0b2e447167b4f5f09ffb26c631 * backport of commit 8c50b0690724ac5e6e243e233db13878c9cf2ef4 * backport of commit e630ec51b386a7fdb87a899e3f82a59f94c99627 --------- Co-authored-by: Ronald Ekambi --- command/acl/token/update/token_update_test.go | 179 ++++++++++-------- 1 file changed, 102 insertions(+), 77 deletions(-) diff --git a/command/acl/token/update/token_update_test.go b/command/acl/token/update/token_update_test.go index 815702841044..011e916f4fbf 100644 --- a/command/acl/token/update/token_update_test.go +++ b/command/acl/token/update/token_update_test.go @@ -22,6 +22,13 @@ func TestTokenUpdateCommand_noTabs(t *testing.T) { } } +func create_token(t *testing.T, client *api.Client, aclToken *api.ACLToken, writeOptions *api.WriteOptions) *api.ACLToken { + token, _, err := client.ACL().TokenCreate(aclToken, writeOptions) + require.NoError(t, err) + + return token +} + func TestTokenUpdateCommand(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") @@ -50,13 +57,6 @@ func TestTokenUpdateCommand(t *testing.T) { ) require.NoError(t, err) - // create a token - token, _, err := client.ACL().TokenCreate( - &api.ACLToken{Description: "test"}, - &api.WriteOptions{Token: "root"}, - ) - require.NoError(t, err) - run := func(t *testing.T, args []string) *api.ACLToken { ui := cli.NewMockUi() cmd := New(ui) @@ -72,7 +72,9 @@ func TestTokenUpdateCommand(t *testing.T) { // update with node identity t.Run("node-identity", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, &api.ACLToken{Description: "test"}, &api.WriteOptions{Token: "root"}) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -80,13 +82,19 @@ func TestTokenUpdateCommand(t *testing.T) { "-description=test token", }) - require.Len(t, token.NodeIdentities, 1) - require.Equal(t, "foo", token.NodeIdentities[0].NodeName) - require.Equal(t, "bar", token.NodeIdentities[0].Datacenter) + require.Len(t, responseToken.NodeIdentities, 1) + require.Equal(t, "foo", responseToken.NodeIdentities[0].NodeName) + require.Equal(t, "bar", responseToken.NodeIdentities[0].Datacenter) }) t.Run("node-identity-merge", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, + client, + &api.ACLToken{Description: "test", NodeIdentities: []*api.ACLNodeIdentity{{NodeName: "foo", Datacenter: "bar"}}}, + &api.WriteOptions{Token: "root"}, + ) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -95,7 +103,7 @@ func TestTokenUpdateCommand(t *testing.T) { "-merge-node-identities", }) - require.Len(t, token.NodeIdentities, 2) + require.Len(t, responseToken.NodeIdentities, 2) expected := []*api.ACLNodeIdentity{ { NodeName: "foo", @@ -106,28 +114,14 @@ func TestTokenUpdateCommand(t *testing.T) { Datacenter: "baz", }, } - require.ElementsMatch(t, expected, token.NodeIdentities) - }) - - // update with append-node-identity - t.Run("append-node-identity", func(t *testing.T) { - - token := run(t, []string{ - "-http-addr=" + a.HTTPAddr(), - "-accessor-id=" + token.AccessorID, - "-token=root", - "-append-node-identity=third:node", - "-description=test token", - }) - - require.Len(t, token.NodeIdentities, 3) - require.Equal(t, "third", token.NodeIdentities[2].NodeName) - require.Equal(t, "node", token.NodeIdentities[2].Datacenter) + require.ElementsMatch(t, expected, responseToken.NodeIdentities) }) // update with policy by name t.Run("policy-name", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, &api.ACLToken{Description: "test"}, &api.WriteOptions{Token: "root"}) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -135,12 +129,14 @@ func TestTokenUpdateCommand(t *testing.T) { "-description=test token", }) - require.Len(t, token.Policies, 1) + require.Len(t, responseToken.Policies, 1) }) // update with policy by id t.Run("policy-id", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, &api.ACLToken{Description: "test"}, &api.WriteOptions{Token: "root"}) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -148,12 +144,14 @@ func TestTokenUpdateCommand(t *testing.T) { "-description=test token", }) - require.Len(t, token.Policies, 1) + require.Len(t, responseToken.Policies, 1) }) // update with service-identity t.Run("service-identity", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, &api.ACLToken{Description: "test"}, &api.WriteOptions{Token: "root"}) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -161,33 +159,22 @@ func TestTokenUpdateCommand(t *testing.T) { "-description=test token", }) - require.Len(t, token.ServiceIdentities, 1) - require.Equal(t, "service", token.ServiceIdentities[0].ServiceName) - }) - - // update with append-service-identity - t.Run("append-service-identity", func(t *testing.T) { - token := run(t, []string{ - "-http-addr=" + a.HTTPAddr(), - "-accessor-id=" + token.AccessorID, - "-token=root", - "-append-service-identity=web", - "-description=test token", - }) - require.Len(t, token.ServiceIdentities, 2) - require.Equal(t, "web", token.ServiceIdentities[1].ServiceName) + require.Len(t, responseToken.ServiceIdentities, 1) + require.Equal(t, "service", responseToken.ServiceIdentities[0].ServiceName) }) // update with no description shouldn't delete the current description t.Run("merge-description", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, &api.ACLToken{Description: "test token"}, &api.WriteOptions{Token: "root"}) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", "-policy-name=" + policy.Name, }) - require.Equal(t, "test token", token.Description) + require.Equal(t, "test token", responseToken.Description) }) } @@ -219,13 +206,6 @@ func TestTokenUpdateCommandWithAppend(t *testing.T) { ) require.NoError(t, err) - // create a token - token, _, err := client.ACL().TokenCreate( - &api.ACLToken{Description: "test", Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}}, - &api.WriteOptions{Token: "root"}, - ) - require.NoError(t, err) - //secondary policy secondPolicy, _, policyErr := client.ACL().PolicyCreate( &api.ACLPolicy{Name: "secondary-policy"}, @@ -233,13 +213,6 @@ func TestTokenUpdateCommandWithAppend(t *testing.T) { ) require.NoError(t, policyErr) - //third policy - thirdPolicy, _, policyErr := client.ACL().PolicyCreate( - &api.ACLPolicy{Name: "third-policy"}, - &api.WriteOptions{Token: "root"}, - ) - require.NoError(t, policyErr) - run := func(t *testing.T, args []string) *api.ACLToken { ui := cli.NewMockUi() cmd := New(ui) @@ -255,7 +228,12 @@ func TestTokenUpdateCommandWithAppend(t *testing.T) { // update with append-policy-name t.Run("append-policy-name", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, + &api.ACLToken{Description: "test", Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}}, + &api.WriteOptions{Token: "root"}, + ) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", @@ -263,20 +241,72 @@ func TestTokenUpdateCommandWithAppend(t *testing.T) { "-description=test token", }) - require.Len(t, token.Policies, 2) + require.Len(t, responseToken.Policies, 2) }) // update with append-policy-id t.Run("append-policy-id", func(t *testing.T) { - token := run(t, []string{ + token := create_token(t, client, + &api.ACLToken{Description: "test", Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}}, + &api.WriteOptions{Token: "root"}, + ) + + responseToken := run(t, []string{ "-http-addr=" + a.HTTPAddr(), "-accessor-id=" + token.AccessorID, "-token=root", - "-append-policy-id=" + thirdPolicy.ID, + "-append-policy-id=" + secondPolicy.ID, "-description=test token", }) - require.Len(t, token.Policies, 3) + require.Len(t, responseToken.Policies, 2) + }) + + // update with append-node-identity + t.Run("append-node-identity", func(t *testing.T) { + token := create_token(t, client, + &api.ACLToken{ + Description: "test", + Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}, + NodeIdentities: []*api.ACLNodeIdentity{{NodeName: "namenode", Datacenter: "somewhere"}}, + }, + &api.WriteOptions{Token: "root"}, + ) + + responseToken := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-node-identity=third:node", + "-description=test token", + }) + + require.Len(t, responseToken.NodeIdentities, 2) + require.Equal(t, "third", responseToken.NodeIdentities[1].NodeName) + require.Equal(t, "node", responseToken.NodeIdentities[1].Datacenter) + }) + + // update with append-service-identity + t.Run("append-service-identity", func(t *testing.T) { + token := create_token(t, client, + &api.ACLToken{ + Description: "test", + Policies: []*api.ACLTokenPolicyLink{{Name: policy.Name}}, + ServiceIdentities: []*api.ACLServiceIdentity{{ServiceName: "service"}}, + }, + &api.WriteOptions{Token: "root"}, + ) + + responseToken := run(t, []string{ + "-http-addr=" + a.HTTPAddr(), + "-accessor-id=" + token.AccessorID, + "-token=root", + "-append-service-identity=web", + "-description=test token", + }) + + require.Len(t, responseToken.ServiceIdentities, 2) + require.Equal(t, "web", responseToken.ServiceIdentities[1].ServiceName) }) } @@ -310,12 +340,7 @@ func TestTokenUpdateCommand_JSON(t *testing.T) { ) require.NoError(t, err) - // create a token - token, _, err := client.ACL().TokenCreate( - &api.ACLToken{Description: "test"}, - &api.WriteOptions{Token: "root"}, - ) - require.NoError(t, err) + token := create_token(t, client, &api.ACLToken{Description: "test"}, &api.WriteOptions{Token: "root"}) t.Run("update with policy by name", func(t *testing.T) { cmd := New(ui) From caf85ac67a01eaf6c9809e97d1059c37f9c1c4b7 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 7 Mar 2023 10:22:16 -0600 Subject: [PATCH 091/421] Backport of Update docs to reflect functionality into release/1.15.x (#16555) * backport of commit 0d101f0e170e0511abceab2159ecb8f21e4fbb22 * backport of commit 140d609194fe08ef8069dc89af9acdf087864cc6 --------- Co-authored-by: Tu Nguyen --- website/content/docs/enterprise/index.mdx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/website/content/docs/enterprise/index.mdx b/website/content/docs/enterprise/index.mdx index d03ab438d208..e63363e81069 100644 --- a/website/content/docs/enterprise/index.mdx +++ b/website/content/docs/enterprise/index.mdx @@ -116,17 +116,17 @@ Consul Enterprise feature availability can change depending on your server and c | Enterprise Feature | VM Client | K8s Client | ECS Client | | ----------------------------------------------------------------------- | :-------: | :--------: | :--------: | -| [Admin Partitions](/consul/docs/enterprise/admin-partitions) | ✅ | ✅ | ❌ | -| [Audit Logging](/consul/docs/enterprise/audit-logging) | ✅ | ✅ | ❌ | -| [Automated Server Backups](/consul/docs/enterprise/backups) | ✅ | ✅ | ❌ | +| [Admin Partitions](/consul/docs/enterprise/admin-partitions) | ✅ | ✅ | ✅ | +| [Audit Logging](/consul/docs/enterprise/audit-logging) | ✅ | ✅ | ✅ | +| [Automated Server Backups](/consul/docs/enterprise/backups) | ✅ | ✅ | ✅ | | [Automated Server Upgrades](/consul/docs/enterprise/upgrades) | ❌ | ❌ | ❌ | | [Enhanced Read Scalability](/consul/docs/enterprise/read-scale) | ❌ | ❌ | ❌ | -| [Namespaces](/consul/docs/enterprise/namespaces) | ✅ | ✅ | ❌ | -| [Network Areas](/consul/docs/enterprise/federation) | ✅ | ✅ | ❌ | +| [Namespaces](/consul/docs/enterprise/namespaces) | ✅ | ✅ | ✅ | +| [Network Areas](/consul/docs/enterprise/federation) | ✅ | ✅ | ✅ | | [Network Segments](/consul/docs/enterprise/network-segments/network-segments-overview) | ❌ | ❌ | ❌ | -| [OIDC Auth Method](/consul/docs/security/acl/auth-methods/oidc) | ✅ | ✅ | ❌ | +| [OIDC Auth Method](/consul/docs/security/acl/auth-methods/oidc) | ✅ | ✅ | ✅ | | [Redundancy Zones](/consul/docs/enterprise/redundancy) | ❌ | ❌ | ❌ | -| [Sentinel ](/consul/docs/enterprise/sentinel) | ✅ | ✅ | ❌ | +| [Sentinel ](/consul/docs/enterprise/sentinel) | ✅ | ✅ | ✅ | From 469705946311d3062734264b4d2de1b16fa5486f Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 7 Mar 2023 11:45:36 -0600 Subject: [PATCH 092/421] Bump consul/sdk to v0.13.1 --- api/go.mod | 4 ++-- api/go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/go.mod b/api/go.mod index c987fde1abef..c8f6faa19cdf 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,11 +2,11 @@ module github.com/hashicorp/consul/api go 1.19 -replace github.com/hashicorp/consul/sdk => ../sdk +//replace github.com/hashicorp/consul/sdk => ../sdk require ( github.com/google/go-cmp v0.5.7 - github.com/hashicorp/consul/sdk v0.13.0 + github.com/hashicorp/consul/sdk v0.13.1 github.com/hashicorp/go-cleanhttp v0.5.1 github.com/hashicorp/go-hclog v0.12.0 github.com/hashicorp/go-rootcerts v1.0.2 diff --git a/api/go.sum b/api/go.sum index 043db4cd1efc..bb2233c5d20f 100644 --- a/api/go.sum +++ b/api/go.sum @@ -15,6 +15,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= +github.com/hashicorp/consul/sdk v0.13.1/go.mod h1:SW/mM4LbKfqmMvcFu8v+eiQQ7oitXEFeiBe9StxERb0= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= From 4afa3a51bdf987b4950376d9d0edccd771d85703 Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 7 Mar 2023 11:56:13 -0600 Subject: [PATCH 093/421] bump consul/api to v1.20.0 and consul/sdk to v0.13.1 --- envoyextensions/go.mod | 6 +++--- envoyextensions/go.sum | 16 ++++------------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/envoyextensions/go.mod b/envoyextensions/go.mod index f96560804c9d..a62847f71326 100644 --- a/envoyextensions/go.mod +++ b/envoyextensions/go.mod @@ -2,12 +2,12 @@ module github.com/hashicorp/consul/envoyextensions go 1.19 -replace github.com/hashicorp/consul/api => ../api +//replace github.com/hashicorp/consul/api => ../api require ( github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 - github.com/hashicorp/consul/api v1.10.1-0.20230209203402-db2bd404bf72 - github.com/hashicorp/consul/sdk v0.13.0 + github.com/hashicorp/consul/api v1.20.0 + github.com/hashicorp/consul/sdk v0.13.1 github.com/hashicorp/go-hclog v1.2.1 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.2.1 diff --git a/envoyextensions/go.sum b/envoyextensions/go.sum index bf542e8f1dc7..95881e292557 100644 --- a/envoyextensions/go.sum +++ b/envoyextensions/go.sum @@ -58,13 +58,14 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/sdk v0.13.0 h1:lce3nFlpv8humJL8rNrrGHYSKc3q+Kxfeg3Ii1m6ZWU= -github.com/hashicorp/consul/sdk v0.13.0/go.mod h1:0hs/l5fOVhJy/VdcoaNqUSi2AUs95eF5WKtv+EYIQqE= +github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc= +github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo= +github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= +github.com/hashicorp/consul/sdk v0.13.1/go.mod h1:SW/mM4LbKfqmMvcFu8v+eiQQ7oitXEFeiBe9StxERb0= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.1 h1:YQsLlGDJgwhXFpucSPyVbCBviQtjlHv3jLTlp8YmtEw= github.com/hashicorp/go-hclog v1.2.1/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= @@ -95,10 +96,7 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -108,7 +106,6 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -124,7 +121,6 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -179,7 +175,6 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -191,7 +186,6 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -236,10 +230,8 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From afc96a6db2abbc4bfa55b5508a146e20432608ad Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 7 Mar 2023 13:31:19 -0600 Subject: [PATCH 094/421] bump consul/api and consul/envoyextensions --- troubleshoot/go.mod | 8 ++++---- troubleshoot/go.sum | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/troubleshoot/go.mod b/troubleshoot/go.mod index c4e445ee05f1..b50703cd9a62 100644 --- a/troubleshoot/go.mod +++ b/troubleshoot/go.mod @@ -2,9 +2,9 @@ module github.com/hashicorp/consul/troubleshoot go 1.19 -replace github.com/hashicorp/consul/api => ../api +//replace github.com/hashicorp/consul/api => ../api -replace github.com/hashicorp/consul/envoyextensions => ../envoyextensions +//replace github.com/hashicorp/consul/envoyextensions => ../envoyextensions exclude ( github.com/hashicorp/go-msgpack v1.1.5 // has breaking changes and must be avoided @@ -13,8 +13,8 @@ exclude ( require ( github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 - github.com/hashicorp/consul/api v1.10.1-0.20230209203402-db2bd404bf72 - github.com/hashicorp/consul/envoyextensions v0.0.0-20230209212012-3b9c56956132 + github.com/hashicorp/consul/api v1.20.0 + github.com/hashicorp/consul/envoyextensions v0.1.2 github.com/stretchr/testify v1.8.0 google.golang.org/protobuf v1.28.1 ) diff --git a/troubleshoot/go.sum b/troubleshoot/go.sum index d2e1e9a2b5b8..a5ce2f255f80 100644 --- a/troubleshoot/go.sum +++ b/troubleshoot/go.sum @@ -83,7 +83,11 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/sdk v0.13.0 h1:lce3nFlpv8humJL8rNrrGHYSKc3q+Kxfeg3Ii1m6ZWU= +github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc= +github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo= +github.com/hashicorp/consul/envoyextensions v0.1.2 h1:PvPqJ/td3UpOeIKQl5ycFPUy46XZP9awfhAUCduDeI4= +github.com/hashicorp/consul/envoyextensions v0.1.2/go.mod h1:N94DQQkgITiA40zuTQ/UdPOLAAWobgHfVT5u7wxE/aU= +github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= From 43e72bddecdcacd248e26191951f4b73675a03ed Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 7 Mar 2023 13:39:58 -0600 Subject: [PATCH 095/421] bump sdk,api,envoyextensions, and troubleshoot modules --- go.mod | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index d591c4dd09a8..2143ed05bb9e 100644 --- a/go.mod +++ b/go.mod @@ -36,11 +36,11 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/consul-awsauth v0.0.0-20220713182709-05ac1c5c2706 github.com/hashicorp/consul-net-rpc v0.0.0-20221205195236-156cfab66a69 - github.com/hashicorp/consul/api v1.18.0 - github.com/hashicorp/consul/envoyextensions v0.0.0-20230209212012-3b9c56956132 + github.com/hashicorp/consul/api v1.20.0 + github.com/hashicorp/consul/envoyextensions v0.1.2 github.com/hashicorp/consul/proto-public v0.2.1 - github.com/hashicorp/consul/sdk v0.13.0 - github.com/hashicorp/consul/troubleshoot v0.0.0-00010101000000-000000000000 + github.com/hashicorp/consul/sdk v0.13.1 + github.com/hashicorp/consul/troubleshoot v0.1.2 github.com/hashicorp/go-bexpr v0.1.2 github.com/hashicorp/go-checkpoint v0.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 From 99e1ccefc2826e2c89b2f41123b5769121c056fe Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Tue, 7 Mar 2023 13:42:33 -0600 Subject: [PATCH 096/421] bump api and sdk modules --- test/integration/consul-container/go.mod | 6 ++---- test/integration/consul-container/go.sum | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 513612707d7c..35174f5fd0b4 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -6,8 +6,8 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/docker/docker v20.10.22+incompatible github.com/docker/go-connections v0.4.0 - github.com/hashicorp/consul/api v1.18.0 - github.com/hashicorp/consul/sdk v0.13.0 + github.com/hashicorp/consul/api v1.20.0 + github.com/hashicorp/consul/sdk v0.13.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.2 @@ -20,7 +20,6 @@ require ( github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 github.com/testcontainers/testcontainers-go v0.15.0 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 - gotest.tools v2.2.0+incompatible ) require ( @@ -39,7 +38,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.0 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-hclog v1.2.1 // indirect diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum index 029b5645d7ce..e554f8ad3f70 100644 --- a/test/integration/consul-container/go.sum +++ b/test/integration/consul-container/go.sum @@ -387,7 +387,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= From fb95d89e4d4cfbaed76c225b7bce68e807339a52 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 8 Mar 2023 11:01:11 -0600 Subject: [PATCH 097/421] backport of commit e5172b9b35c9cd18b9eb501a39d7923a1af080d5 (#16573) Co-authored-by: Eric --- .changelog/16570.txt | 3 ++ agent/consul/state/catalog.go | 31 ++++++++++-------- agent/consul/state/catalog_test.go | 51 +++++++++++++++++++++++------- agent/consul/state/usage.go | 6 ++-- agent/structs/structs.go | 12 ++++--- 5 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 .changelog/16570.txt diff --git a/.changelog/16570.txt b/.changelog/16570.txt new file mode 100644 index 000000000000..ad07cda81c06 --- /dev/null +++ b/.changelog/16570.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fixes a bug that can lead to peering service deletes impacting the state of local services +``` diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 39a40d9cad36..077fbde79a83 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -1186,7 +1186,7 @@ func serviceListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, unique := make(map[structs.ServiceName]struct{}) for service := services.Next(); service != nil; service = services.Next() { svc := service.(*structs.ServiceNode) - unique[svc.CompoundServiceName()] = struct{}{} + unique[svc.CompoundServiceName().ServiceName] = struct{}{} } results := make(structs.ServiceList, 0, len(unique)) @@ -1920,17 +1920,17 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st return fmt.Errorf("failed updating service-kind indexes: %w", err) } // Update the node indexes as the service information is included in node catalog queries. - if err := catalogUpdateNodesIndexes(tx, idx, entMeta, peerName); err != nil { + if err := catalogUpdateNodesIndexes(tx, idx, entMeta, svc.PeerName); err != nil { return fmt.Errorf("failed updating nodes indexes: %w", err) } - if err := catalogUpdateNodeIndexes(tx, idx, nodeName, entMeta, peerName); err != nil { + if err := catalogUpdateNodeIndexes(tx, idx, nodeName, entMeta, svc.PeerName); err != nil { return fmt.Errorf("failed updating node indexes: %w", err) } - name := svc.CompoundServiceName() + psn := svc.CompoundServiceName() if err := cleanupMeshTopology(tx, idx, svc); err != nil { - return fmt.Errorf("failed to clean up mesh-topology associations for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up mesh-topology associations for %q: %v", psn.String(), err) } q := Query{ @@ -1957,12 +1957,14 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st if err := catalogUpdateServiceExtinctionIndex(tx, idx, entMeta, svc.PeerName); err != nil { return err } - psn := structs.PeeredServiceName{Peer: svc.PeerName, ServiceName: name} if err := freeServiceVirtualIP(tx, idx, psn, nil); err != nil { - return fmt.Errorf("failed to clean up virtual IP for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up virtual IP for %q: %v", psn.String(), err) } - if err := cleanupKindServiceName(tx, idx, svc.CompoundServiceName(), svc.ServiceKind); err != nil { - return fmt.Errorf("failed to persist service name: %v", err) + + if svc.PeerName == "" { + if err := cleanupKindServiceName(tx, idx, psn.ServiceName, svc.ServiceKind); err != nil { + return fmt.Errorf("failed to persist service name: %v", err) + } } } } else { @@ -1990,7 +1992,7 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st if svc.PeerName == "" { sn := structs.ServiceName{Name: svc.ServiceName, EnterpriseMeta: svc.EnterpriseMeta} if err := cleanupGatewayWildcards(tx, idx, sn, false); err != nil { - return fmt.Errorf("failed to clean up gateway-service associations for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up gateway-service associations for %q: %v", psn.String(), err) } } @@ -2695,7 +2697,7 @@ func (s *Store) CheckIngressServiceNodes(ws memdb.WatchSet, serviceName string, // De-dup services to lookup names := make(map[structs.ServiceName]struct{}) for _, n := range nodes { - names[n.CompoundServiceName()] = struct{}{} + names[n.CompoundServiceName().ServiceName] = struct{}{} } var results structs.CheckServiceNodes @@ -3657,7 +3659,7 @@ func updateGatewayNamespace(tx WriteTxn, idx uint64, service *structs.GatewaySer continue } - existing, err := tx.First(tableGatewayServices, indexID, service.Gateway, sn.CompoundServiceName(), service.Port) + existing, err := tx.First(tableGatewayServices, indexID, service.Gateway, sn.CompoundServiceName().ServiceName, service.Port) if err != nil { return fmt.Errorf("gateway service lookup failed: %s", err) } @@ -4611,7 +4613,10 @@ func updateMeshTopology(tx WriteTxn, idx uint64, node string, svc *structs.NodeS // cleanupMeshTopology removes a service from the mesh topology table // This is only safe to call when there are no more known instances of this proxy func cleanupMeshTopology(tx WriteTxn, idx uint64, service *structs.ServiceNode) error { - // TODO(peering): make this peering aware? + if service.PeerName != "" { + return nil + } + if service.ServiceKind != structs.ServiceKindConnectProxy { return nil } diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index cef5ba0a0a07..b5976bbd2b00 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -2577,20 +2577,49 @@ func TestStateStore_DeleteService(t *testing.T) { testRegisterService(t, s, 2, "node1", "service1") testRegisterCheck(t, s, 3, "node1", "service1", "check1", api.HealthPassing) - // Delete the service. + // register a node with a service on a cluster peer. + testRegisterNodeOpts(t, s, 4, "node1", func(n *structs.Node) error { + n.PeerName = "cluster-01" + return nil + }) + testRegisterServiceOpts(t, s, 5, "node1", "service1", func(service *structs.NodeService) { + service.PeerName = "cluster-01" + }) + + wsPeer := memdb.NewWatchSet() + _, ns, err := s.NodeServices(wsPeer, "node1", nil, "cluster-01") + require.Len(t, ns.Services, 1) + require.NoError(t, err) + ws := memdb.NewWatchSet() - _, _, err := s.NodeServices(ws, "node1", nil, "") + _, ns, err = s.NodeServices(ws, "node1", nil, "") + require.Len(t, ns.Services, 1) require.NoError(t, err) - if err := s.DeleteService(4, "node1", "service1", nil, ""); err != nil { - t.Fatalf("err: %s", err) + + { + // Delete the peered service. + err = s.DeleteService(6, "node1", "service1", nil, "cluster-01") + require.NoError(t, err) + require.True(t, watchFired(wsPeer)) + _, kindServiceNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindTypical) + require.NoError(t, err) + require.Len(t, kindServiceNames, 1) + require.Equal(t, "service1", kindServiceNames[0].Service.Name) } - if !watchFired(ws) { - t.Fatalf("bad") + + { + // Delete the service. + err = s.DeleteService(6, "node1", "service1", nil, "") + require.NoError(t, err) + require.True(t, watchFired(ws)) + _, kindServiceNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindTypical) + require.NoError(t, err) + require.Len(t, kindServiceNames, 0) } // Service doesn't exist. ws = memdb.NewWatchSet() - _, ns, err := s.NodeServices(ws, "node1", nil, "") + _, ns, err = s.NodeServices(ws, "node1", nil, "") if err != nil || ns == nil || len(ns.Services) != 0 { t.Fatalf("bad: %#v (err: %#v)", ns, err) } @@ -2605,15 +2634,15 @@ func TestStateStore_DeleteService(t *testing.T) { } // Index tables were updated. - assert.Equal(t, uint64(4), catalogChecksMaxIndex(tx, nil, "")) - assert.Equal(t, uint64(4), catalogServicesMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogChecksMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogServicesMaxIndex(tx, nil, "")) // Deleting a nonexistent service should be idempotent and not return an // error, nor fire a watch. - if err := s.DeleteService(5, "node1", "service1", nil, ""); err != nil { + if err := s.DeleteService(6, "node1", "service1", nil, ""); err != nil { t.Fatalf("err: %s", err) } - assert.Equal(t, uint64(4), catalogServicesMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogServicesMaxIndex(tx, nil, "")) if watchFired(ws) { t.Fatalf("bad") } diff --git a/agent/consul/state/usage.go b/agent/consul/state/usage.go index 5e3d7ce9cb37..9db30e5b2d16 100644 --- a/agent/consul/state/usage.go +++ b/agent/consul/state/usage.go @@ -126,11 +126,11 @@ func updateUsage(tx WriteTxn, changes Changes) error { // changed, in order to compare it with the finished memdb state. // Make sure to account for the fact that services can change their names. if serviceNameChanged(change) { - serviceNameChanges[svc.CompoundServiceName()] += 1 + serviceNameChanges[svc.CompoundServiceName().ServiceName] += 1 before := change.Before.(*structs.ServiceNode) - serviceNameChanges[before.CompoundServiceName()] -= 1 + serviceNameChanges[before.CompoundServiceName().ServiceName] -= 1 } else { - serviceNameChanges[svc.CompoundServiceName()] += delta + serviceNameChanges[svc.CompoundServiceName().ServiceName] += delta } case "kvs": diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 5191272a645f..117de8542341 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -1153,7 +1153,7 @@ func (sn *ServiceNode) CompoundServiceID() ServiceID { } } -func (sn *ServiceNode) CompoundServiceName() ServiceName { +func (sn *ServiceNode) CompoundServiceName() PeeredServiceName { name := sn.ServiceName if name == "" { name = sn.ServiceID @@ -1163,10 +1163,14 @@ func (sn *ServiceNode) CompoundServiceName() ServiceName { entMeta := sn.EnterpriseMeta entMeta.Normalize() - return ServiceName{ - Name: name, - EnterpriseMeta: entMeta, + return PeeredServiceName{ + ServiceName: ServiceName{ + Name: name, + EnterpriseMeta: entMeta, + }, + Peer: sn.PeerName, } + } // Weights represent the weight used by DNS for a given status From 4839df749e9c11107b0c909e36d6f782a1fc08ab Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Wed, 8 Mar 2023 13:21:51 -0600 Subject: [PATCH 098/421] Update changelog for 1.15.1 patch release --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a9c47514261..485d26865235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,33 @@ +## 1.15.1 (March 7, 2023) + +IMPROVEMENTS: + +* cli: added `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id` flags to the `consul token update` command. +These flags allow updates to a token's policies/roles without having to override them completely. [[GH-16288](https://github.com/hashicorp/consul/issues/16288)] +* cli: added `-append-service-identity` and `-append-node-identity` flags to the `consul token update` command. +These flags allow updates to a token's node identities/service identities without having to override them. [[GH-16506](https://github.com/hashicorp/consul/issues/16506)] +* connect: Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 [[GH-16274](https://github.com/hashicorp/consul/issues/16274)] +* mesh: Add ServiceResolver RequestTimeout for route timeouts to make request timeouts configurable [[GH-16495](https://github.com/hashicorp/consul/issues/16495)] +* ui: support filtering API gateways in the ui and displaying their documentation links [[GH-16508](https://github.com/hashicorp/consul/issues/16508)] + +DEPRECATIONS: + +* cli: Deprecate the `-merge-node-identites` and `-merge-service-identities` flags from the `consul token update` command in favor of: `-append-node-identity` and `-append-service-identity`. [[GH-16506](https://github.com/hashicorp/consul/issues/16506)] +* cli: Deprecate the `-merge-policies` and `-merge-roles` flags from the `consul token update` command in favor of: `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id`. [[GH-16288](https://github.com/hashicorp/consul/issues/16288)] + +BUG FIXES: + +* cli: Fixes an issue with `consul connect envoy` where a log to STDOUT could malform JSON when used with `-bootstrap`. [[GH-16530](https://github.com/hashicorp/consul/issues/16530)] +* cli: Fixes an issue with `consul connect envoy` where grpc-disabled agents were not error-handled correctly. [[GH-16530](https://github.com/hashicorp/consul/issues/16530)] +* cli: ensure acl token read -self works [[GH-16445](https://github.com/hashicorp/consul/issues/16445)] +* cli: fix panic read non-existent acl policy [[GH-16485](https://github.com/hashicorp/consul/issues/16485)] +* gateways: fix HTTPRoute bug where service weights could be less than or equal to 0 and result in a downstream envoy protocol error [[GH-16512](https://github.com/hashicorp/consul/issues/16512)] +* gateways: fix HTTPRoute bug where services with a weight not divisible by 10000 are never registered properly [[GH-16531](https://github.com/hashicorp/consul/issues/16531)] +* mesh: Fix resolution of service resolvers with subsets for external upstreams [[GH-16499](https://github.com/hashicorp/consul/issues/16499)] +* proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher [[GH-16497](https://github.com/hashicorp/consul/issues/16497)] +* proxycfg: fix a bug where terminating gateways were not cleaning up deleted service resolvers for their referenced services [[GH-16498](https://github.com/hashicorp/consul/issues/16498)] +* ui: Fix issue with lists and filters not rendering properly [[GH-16444](https://github.com/hashicorp/consul/issues/16444)] + ## 1.15.0 (February 23, 2023) BREAKING CHANGES: From 5b20c1a29ceb9a6c56ecfb495ccfeda25bb39715 Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Wed, 8 Mar 2023 13:27:22 -0600 Subject: [PATCH 099/421] Bump version for next release --- version/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/VERSION b/version/VERSION index bf222ecb47b4..827d8d6a3bfa 100644 --- a/version/VERSION +++ b/version/VERSION @@ -1 +1 @@ -1.15.1-dev \ No newline at end of file +1.15.2-dev From bf7dc8d3e25d67bc3ec4453bf98127d7c3266ac6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 9 Mar 2023 11:29:08 -0600 Subject: [PATCH 100/421] backport of commit 6fc60982313384ff1ef9d2aed9ce50070f79102b (#16586) Co-authored-by: Paul Glass --- website/content/api-docs/config.mdx | 157 +++++++++++++++++----------- 1 file changed, 96 insertions(+), 61 deletions(-) diff --git a/website/content/api-docs/config.mdx b/website/content/api-docs/config.mdx index 3c49e0c8d347..96e6a7b4de77 100644 --- a/website/content/api-docs/config.mdx +++ b/website/content/api-docs/config.mdx @@ -31,25 +31,32 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------------------------- | -| `NO` | `none` | `none` | `service:write`
    `operator:write`1 | - -

    - 1 The ACL required depends on the config entry kind being updated: -

    - -| Config Entry Kind | Required ACL | -| ------------------- | ------------------ | -| ingress-gateway | `operator:write` | -| proxy-defaults | `operator:write` | -| service-defaults | `service:write` | -| service-intentions | `intentions:write` | -| service-resolver | `service:write` | -| service-router | `service:write` | -| service-splitter | `service:write` | -| terminating-gateway | `operator:write` | +| `NO` | `none` | `none` | Refer to [Permissions](#permissions) | The corresponding CLI command is [`consul config write`](/consul/commands/config/write). +### Permissions + +The ACL required depends on the config entry being written: + +| Config Entry Kind | Required ACLs | +| ------------------- | -------------------------------- | +| api-gateway | `mesh:write` or `operator:write` | +| bound-api-gateway | Not writable. | +| exported-services | `mesh:write` or `operator:write` | +| http-route | `mesh:write` or `operator:write` | +| ingress-gateway | `mesh:write` or `operator:write` | +| inline-certificate | `mesh:write` or `operator:write` | +| mesh | `mesh:write` or `operator:write` | +| proxy-defaults | `mesh:write` or `operator:write` | +| service-defaults | `service:write` | +| service-intentions | `intentions:write` | +| service-resolver | `service:write` | +| service-router | `service:write` | +| service-splitter | `service:write` | +| tcp-route | `mesh:write` or `operator:write` | +| terminating-gateway | `mesh:write` or `operator:write` | + ### Query Parameters - `dc` `(string: "")` - Specifies the datacenter to query. @@ -96,25 +103,35 @@ The table below shows this endpoint's support for [agent caching](/consul/api-docs/features/caching), and [required ACLs](/consul/api-docs/api-structure#authentication). -| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | -| ---------------- | ----------------- | ------------- | -------------------------- | -| `YES` | `all` | `none` | `service:read`1 | - -1 The ACL required depends on the config entry kind being read: +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | ------------- | -------------------------------------- | +| `YES` | `all` | `none` | Refer to [Permissions](#permissions-1) | -| Config Entry Kind | Required ACL | -| ------------------- | ----------------- | -| ingress-gateway | `service:read` | -| proxy-defaults | `` | -| service-defaults | `service:read` | -| service-intentions | `intentions:read` | -| service-resolver | `service:read` | -| service-router | `service:read` | -| service-splitter | `service:read` | -| terminating-gateway | `service:read` | The corresponding CLI command is [`consul config read`](/consul/commands/config/read). +### Permissions + +The ACL required depends on the config entry kind being read: + +| Config Entry Kind | Required ACLs | +| ------------------- | -------------------------------- | +| api-gateway | `service:read` | +| bound-api-gateway | `service:read` | +| exported-services | `mesh:read` or `operator:read` | +| http-route | `mesh:read` or `operator:read` | +| ingress-gateway | `service:read` | +| inline-certificate | `mesh:read` or `operator:read` | +| mesh | No ACL required | +| proxy-defaults | No ACL required | +| service-defaults | `service:read` | +| service-intentions | `intentions:read` | +| service-resolver | `service:read` | +| service-router | `service:read` | +| service-splitter | `service:read` | +| tcp-route | `mesh:read` or `operator:read` | +| terminating-gateway | `service:read` | + ### Path Parameters - `kind` `(string: )` - Specifies the kind of the entry to read. @@ -167,22 +184,31 @@ The table below shows this endpoint's support for [agent caching](/consul/api-docs/features/caching), and [required ACLs](/consul/api-docs/api-structure#authentication). -| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | -| ---------------- | ----------------- | ------------- | -------------------------- | -| `YES` | `all` | `none` | `service:read`1 | - -1 The ACL required depends on the config entry kind being read: - -| Config Entry Kind | Required ACL | -| ------------------- | ----------------- | -| ingress-gateway | `service:read` | -| proxy-defaults | `` | -| service-defaults | `service:read` | -| service-intentions | `intentions:read` | -| service-resolver | `service:read` | -| service-router | `service:read` | -| service-splitter | `service:read` | -| terminating-gateway | `service:read` | +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | ------------- | -------------------------------------- | +| `YES` | `all` | `none` | Refer to [Permissions](#permissions-2) | + +### Permissions + +The ACL required depends on the config entry kind being read: + +| Config Entry Kind | Required ACLs | +| ------------------- | -------------------------------- | +| api-gateway | `service:read` | +| bound-api-gateway | `service:read` | +| exported-services | `mesh:read` or `operator:read` | +| http-route | `mesh:read` or `operator:read` | +| ingress-gateway | `service:read` | +| inline-certificate | `mesh:read` or `operator:read` | +| mesh | No ACL required | +| proxy-defaults | No ACL required | +| service-defaults | `service:read` | +| service-intentions | `intentions:read` | +| service-resolver | `service:read` | +| service-router | `service:read` | +| service-splitter | `service:read` | +| tcp-route | `mesh:read` or `operator:read` | +| terminating-gateway | `service:read` | The corresponding CLI command is [`consul config list`](/consul/commands/config/list). @@ -243,20 +269,29 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------------------------- | -| `NO` | `none` | `none` | `service:write`
    `operator:write`1 | - -1 The ACL required depends on the config entry kind being deleted: - -| Config Entry Kind | Required ACL | -| ------------------- | ------------------ | -| ingress-gateway | `operator:write` | -| proxy-defaults | `operator:write` | -| service-defaults | `service:write` | -| service-intentions | `intentions:write` | -| service-resolver | `service:write` | -| service-router | `service:write` | -| service-splitter | `service:write` | -| terminating-gateway | `operator:write ` | +| `NO` | `none` | `none` | Refer to [Permissions](#permissions-3) | + +### Permissions + +The ACL required depends on the config entry kind being deleted: + +| Config Entry Kind | Required ACLs | +| ------------------- | -------------------------------- | +| api-gateway | `mesh:write` or `operator:write` | +| bound-api-gateway | Not writable. | +| exported-services | `mesh:write` or `operator:write` | +| http-route | `mesh:write` or `operator:write` | +| ingress-gateway | `mesh:write` or `operator:write` | +| inline-certificate | `mesh:write` or `operator:write` | +| mesh | `mesh:write` or `operator:write` | +| proxy-defaults | `mesh:write` or `operator:write` | +| service-defaults | `service:write` | +| service-intentions | `intentions:write` | +| service-resolver | `service:write` | +| service-router | `service:write` | +| service-splitter | `service:write` | +| tcp-route | `mesh:write` or `operator:write` | +| terminating-gateway | `mesh:write` or `operator:write` | The corresponding CLI command is [`consul config delete`](/consul/commands/config/delete). From 57ad7e059e6693630b8c4202534da41fa0945a0a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 9 Mar 2023 14:40:18 -0600 Subject: [PATCH 101/421] Backport of added a backport-checker GitHub action into release/1.15.x (#16584) * backport of commit 15c255a2e802609dbc15deedbfb00bf65668594b * backport of commit 49b51848715b420aab69cd7b9bed61cec04f06cc --------- Co-authored-by: Michael Wilkerson Co-authored-by: Michael Wilkerson <62034708+wilkermichael@users.noreply.github.com> --- .github/workflows/backport-checker.yml | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/backport-checker.yml diff --git a/.github/workflows/backport-checker.yml b/.github/workflows/backport-checker.yml new file mode 100644 index 000000000000..c9af23ea9717 --- /dev/null +++ b/.github/workflows/backport-checker.yml @@ -0,0 +1,32 @@ +# This workflow checks that there is either a 'pr/no-backport' label applied to a PR +# or there is a backport/* label indicating a backport has been set + +name: Backport Checker + +on: + pull_request: + types: [opened, synchronize, labeled] + # Runs on PRs to main and all release branches + branches: + - main + - release/* + +jobs: + # checks that a backport label is present for a PR + backport-check: + # If there's a `pr/no-backport` label we ignore this check. Also, we ignore PRs created by the bot assigned to `backport-assistant` + if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-backport') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" + runs-on: ubuntu-latest + + steps: + - name: Check for Backport Label + run: | + labels="${{join(github.event.pull_request.labels.*.name, ', ') }}" + if [[ "$labels" =~ .*"backport/".* ]]; then + echo "Found backport label!" + exit 0 + fi + # Fail status check when no backport label was found on the PR + echo "Did not find a backport label matching the pattern 'backport/*' and the 'pr/no-backport' label was not applied. Reference - https://github.com/hashicorp/consul/pull/16567" + exit 1 + From 1cd8b6fb8c1188cd8bd35c80bbd47105e25fd1e1 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 9 Mar 2023 16:31:16 -0600 Subject: [PATCH 102/421] Backport of JIRA pr check: Filter out OSS/ENT merges into release/1.15.x (#16594) * backport of commit c7998faf73a688d3398d18a4332ed054c24294c0 * backport of commit f38a1d4a2fb0503873389fcae35423eaf626dd19 --------- Co-authored-by: David Yu --- .github/workflows/jira-pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/jira-pr.yaml b/.github/workflows/jira-pr.yaml index 17b7d26b1ff8..be8eb77865bb 100644 --- a/.github/workflows/jira-pr.yaml +++ b/.github/workflows/jira-pr.yaml @@ -41,6 +41,12 @@ jobs: if [[ -n ${ROLE} ]]; then echo "Actor ${{ github.actor }} is a ${TEAM} team member" echo "MESSAGE=true" >> $GITHUB_OUTPUT + elif [[ "${{ contains(github.actor, 'hc-github-team-consul-core') }}" == "true" ]]; then + echo "Actor ${{ github.actor }} is a ${TEAM} team member" + echo "MESSAGE=true" >> $GITHUB_OUTPUT + elif [[ "${{ contains(github.actor, 'dependabot') }}" == "true" ]]; then + echo "Actor ${{ github.actor }} is a ${TEAM} team member" + echo "MESSAGE=true" >> $GITHUB_OUTPUT else echo "Actor ${{ github.actor }} is NOT a ${TEAM} team member" echo "MESSAGE=false" >> $GITHUB_OUTPUT From 71a3e9f3524cc82f5f9464ca502481c3871931e7 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 10 Mar 2023 13:34:32 -0600 Subject: [PATCH 103/421] Backport of fixes for unsupported partitions field in CRD metadata block into release/1.15.x (#16606) * backport of commit b7eae1ebb3b2948d07bf5930e278453ddd6eb9e7 * backport of commit 32bdd2cc35c653d70e466f0ef7b4576ac203bd3f --------- Co-authored-by: trujillo-adam Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --- .../config-entries/ingress-gateway.mdx | 4 +- .../config-entries/service-defaults.mdx | 9 -- .../docs/services/usage/define-services.mdx | 86 ++++++++++++++++++- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/website/content/docs/connect/config-entries/ingress-gateway.mdx b/website/content/docs/connect/config-entries/ingress-gateway.mdx index b1cb5c87ab49..07f01a03b1c4 100644 --- a/website/content/docs/connect/config-entries/ingress-gateway.mdx +++ b/website/content/docs/connect/config-entries/ingress-gateway.mdx @@ -88,6 +88,9 @@ spec: + +For Kubernetes environments, the configuration entry is always created in the same partition as the Kubernetes cluster. + ```hcl @@ -117,7 +120,6 @@ kind: IngressGateway metadata: name: namespace: - partition: spec: listeners: diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 3dd9812d3c0d..f14b0c63e643 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -82,7 +82,6 @@ The following outline shows how to format the service splitter configuration ent - [`metadata`](#metadata): map | no default - [`name`](#name): string | no default - [`namespace`](#namespace): string | no default | - - [`partition`](#partition): string | no default | - [`spec`](#spec): map | no default - [`protocol`](#protocol): string | default: `tcp` - [`balanceInboundConnections`](#balanceinboundconnections): string | no default @@ -239,7 +238,6 @@ kind: ServiceDefaults metadata: name: namespace: - partition: spec: protocol: tcp balanceInboundConnnections: exact_balance @@ -802,13 +800,6 @@ Specifies the Consul namespace that the configuration entry applies to. Refer to - Default: `default` - Data type: string -### `metadata.partition` - -Specifies the name of the name of the Consul admin partition that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul Enterprise on Kubernetes. Consul OSS distributions ignore the `metadata.partition` configuration. - -- Default: `default` -- Data type: string - ### `spec` Map that contains the details about the `ServiceDefaults` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the `spec` field. All other configurations are children. diff --git a/website/content/docs/services/usage/define-services.mdx b/website/content/docs/services/usage/define-services.mdx index a4b6eaa15961..3a7862c92a73 100644 --- a/website/content/docs/services/usage/define-services.mdx +++ b/website/content/docs/services/usage/define-services.mdx @@ -15,7 +15,7 @@ You must tell Consul about the services deployed to your network if you want the You can define multiple services individually using `service` blocks or group multiple services into the same `services` configuration block. Refer to [Define multiple services in a single file](#define-multiple-services-in-a-single-file) for additional information. -If Consul service mesh is enabled in your network, you can use the `service-defaults` configuration entry to specify default global values for services. The configuraiton entry lets you define common service parameter, such as upstreams, namespaces, and partitions. Refer to [Define service defaults](#define-service-defaults) for additional information. +If Consul service mesh is enabled in your network, you can use the [service defaults configuration entry](/consul/docs/connect/config-entries/service-defaults) to specify default global values for services. The configuration entry lets you define common service parameter, such as upstreams, namespaces, and partitions. Refer to [Define service defaults](#define-service-defaults) for additional information. ## Requirements @@ -145,6 +145,9 @@ If Consul service mesh is enabled in your network, you can define default values Create a file for the configuration entry and specify the required fields. If you are authoring `service-defaults` in HCL or JSON, the `Kind` and `Name` fields are required. On Kubernetes, the `apiVersion`, `kind`, and `metadata.name` fields are required. Refer to [Service Defaults Reference](/consul/docs/connect/config-entries/service-defaults) for details about the configuration options. +If you use Consul Enterprise, you can also specify the `Namespace` and `Partition` fields to apply the configuration to services in a specific namespace or partition. For Kubernetes environments, the configuration entry is always created in the same partition as the Kubernetes cluster. + +### Consul OSS example The following example instructs services named `counting` to send up to `512` concurrent requests to a mesh gateway: @@ -222,6 +225,87 @@ spec: ``` +### Consul Enterprise example +The following example instructs services named `counting` in the `prod` namespace to send up to `512` concurrent requests to a mesh gateway: + + + +```hcl +Kind = "service-defaults" +Name = "counting" +Namespace = "prod" + +UpstreamConfig = { + Defaults = { + MeshGateway = { + Mode = "local" + } + Limits = { + MaxConnections = 512 + MaxPendingRequests = 512 + MaxConcurrentRequests = 512 + } + } + + Overrides = [ + { + Name = "dashboard" + MeshGateway = { + Mode = "remote" + } + } + ] +} +``` +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: counting + namespace: prod +spec: + upstreamConfig: + defaults: + meshGateway: + mode: local + limits: + maxConnections: 512 + maxPendingRequests: 512 + maxConcurrentRequests: 512 + overrides: + - name: dashboard + meshGateway: + mode: remote +``` +```json +{ + "Kind": "service-defaults", + "Name": "counting", + "Namespace" : "prod", + "UpstreamConfig": { + "Defaults": { + "MeshGateway": { + "Mode": "local" + }, + "Limits": { + "MaxConnections": 512, + "MaxPendingRequests": 512, + "MaxConcurrentRequests": 512 + } + }, + "Overrides": [ + { + "Name": "dashboard", + "MeshGateway": { + "Mode": "remote" + } + } + ] + } +} +``` + + ### Apply service defaults You can apply your `service-defaults` configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). In Kubernetes environments, apply the `service-defaults` custom resource definitions (CRD) to implement and manage Consul configuration entries. From 02f8ed4ca2526f8b4e6b23e2a42b91df09bd381f Mon Sep 17 00:00:00 2001 From: Freddy Date: Fri, 10 Mar 2023 14:50:30 -0700 Subject: [PATCH 104/421] Backport of Allow HCP metrics collection for Envoy proxies into release/1.15.x (#16611) Co-authored-by: Ashvitha Sridharan Co-authored-by: Freddy Co-authored-by: Ashvitha --- .changelog/16585.txt | 3 + agent/proxycfg/connect_proxy.go | 71 +++++ agent/proxycfg/state_test.go | 181 ++++++++++++- agent/proxycfg/testing_connect_proxy.go | 50 ++++ agent/xds/resources_test.go | 4 + .../clusters/hcp-metrics.latest.golden | 183 +++++++++++++ .../endpoints/hcp-metrics.latest.golden | 97 +++++++ .../listeners/hcp-metrics.latest.golden | 184 +++++++++++++ .../testdata/routes/hcp-metrics.latest.golden | 5 + .../secrets/hcp-metrics.latest.golden | 5 + api/connect.go | 3 + command/connect/envoy/bootstrap_config.go | 66 ++++- .../connect/envoy/bootstrap_config_test.go | 200 ++++++++++++-- command/connect/envoy/bootstrap_tpl.go | 4 +- command/connect/envoy/envoy_test.go | 23 ++ .../connect/envoy/testdata/hcp-metrics.golden | 247 ++++++++++++++++++ website/content/commands/connect/envoy.mdx | 4 + 17 files changed, 1293 insertions(+), 37 deletions(-) create mode 100644 .changelog/16585.txt create mode 100644 agent/xds/testdata/clusters/hcp-metrics.latest.golden create mode 100644 agent/xds/testdata/endpoints/hcp-metrics.latest.golden create mode 100644 agent/xds/testdata/listeners/hcp-metrics.latest.golden create mode 100644 agent/xds/testdata/routes/hcp-metrics.latest.golden create mode 100644 agent/xds/testdata/secrets/hcp-metrics.latest.golden create mode 100644 command/connect/envoy/testdata/hcp-metrics.golden diff --git a/.changelog/16585.txt b/.changelog/16585.txt new file mode 100644 index 000000000000..11e2959cfbdb --- /dev/null +++ b/.changelog/16585.txt @@ -0,0 +1,3 @@ +```release-note:feature +xds: Allow for configuring connect proxies to send service mesh telemetry to an HCP metrics collection service. +``` \ No newline at end of file diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index 81b41db69401..813f5bdc960f 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -3,12 +3,16 @@ package proxycfg import ( "context" "fmt" + "path" "strings" + "github.com/hashicorp/consul/acl" cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/proxycfg/internal/watch" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/proto/pbpeering" + "github.com/mitchellh/mapstructure" ) type handlerConnectProxy struct { @@ -103,6 +107,10 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e return snap, err } + if err := s.maybeInitializeHCPMetricsWatches(ctx, snap); err != nil { + return snap, fmt.Errorf("failed to initialize HCP metrics watches: %w", err) + } + if s.proxyCfg.Mode == structs.ProxyModeTransparent { // When in transparent proxy we will infer upstreams from intentions with this source err := s.dataSources.IntentionUpstreams.Notify(ctx, &structs.ServiceSpecificRequest{ @@ -614,3 +622,66 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s } return nil } + +// hcpMetricsConfig represents the basic opaque config values for pushing telemetry to HCP. +type hcpMetricsConfig struct { + // HCPMetricsBindSocketDir is a string that configures the directory for a + // unix socket where Envoy will forward metrics. These metrics get pushed to + // the HCP Metrics collector to show service mesh metrics on HCP. + HCPMetricsBindSocketDir string `mapstructure:"envoy_hcp_metrics_bind_socket_dir"` +} + +func parseHCPMetricsConfig(m map[string]interface{}) (hcpMetricsConfig, error) { + var cfg hcpMetricsConfig + err := mapstructure.WeakDecode(m, &cfg) + + if err != nil { + return cfg, fmt.Errorf("failed to decode: %w", err) + } + + return cfg, nil +} + +// maybeInitializeHCPMetricsWatches will initialize a synthetic upstream and discovery chain +// watch for the HCP metrics collector, if metrics collection is enabled on the proxy registration. +func (s *handlerConnectProxy) maybeInitializeHCPMetricsWatches(ctx context.Context, snap ConfigSnapshot) error { + hcpCfg, err := parseHCPMetricsConfig(s.proxyCfg.Config) + if err != nil { + s.logger.Error("failed to parse connect.proxy.config", "error", err) + } + + if hcpCfg.HCPMetricsBindSocketDir == "" { + // Metrics collection is not enabled, return early. + return nil + } + + // The path includes the proxy ID so that when multiple proxies are on the same host + // they each have a distinct path to send their metrics. + sock := fmt.Sprintf("%s_%s.sock", s.proxyID.NamespaceOrDefault(), s.proxyID.ID) + path := path.Join(hcpCfg.HCPMetricsBindSocketDir, sock) + + upstream := structs.Upstream{ + DestinationNamespace: acl.DefaultNamespaceName, + DestinationPartition: s.proxyID.PartitionOrDefault(), + DestinationName: api.HCPMetricsCollectorName, + LocalBindSocketPath: path, + Config: map[string]interface{}{ + "protocol": "grpc", + }, + } + uid := NewUpstreamID(&upstream) + snap.ConnectProxy.UpstreamConfig[uid] = &upstream + + err = s.dataSources.CompiledDiscoveryChain.Notify(ctx, &structs.DiscoveryChainRequest{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + Name: upstream.DestinationName, + EvaluateInDatacenter: s.source.Datacenter, + EvaluateInNamespace: uid.NamespaceOrDefault(), + EvaluateInPartition: uid.PartitionOrDefault(), + }, "discovery-chain:"+uid.String(), s.ch) + if err != nil { + return fmt.Errorf("failed to watch discovery chain for %s: %v", uid.String(), err) + } + return nil +} diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index bdb2e3c07bc7..b83edb4af46c 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -15,6 +15,7 @@ import ( cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" + apimod "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/consul/proto/prototest" "github.com/hashicorp/consul/sdk/testutil" @@ -455,16 +456,18 @@ func TestState_WatchesAndUpdates(t *testing.T) { // Used to account for differences in OSS/ent implementations of ServiceID.String() var ( - db = structs.NewServiceName("db", nil) - billing = structs.NewServiceName("billing", nil) - api = structs.NewServiceName("api", nil) - apiA = structs.NewServiceName("api-a", nil) - - apiUID = NewUpstreamIDFromServiceName(api) - dbUID = NewUpstreamIDFromServiceName(db) - pqUID = UpstreamIDFromString("prepared_query:query") - extApiUID = NewUpstreamIDFromServiceName(apiA) - extDBUID = NewUpstreamIDFromServiceName(db) + db = structs.NewServiceName("db", nil) + billing = structs.NewServiceName("billing", nil) + api = structs.NewServiceName("api", nil) + apiA = structs.NewServiceName("api-a", nil) + hcpCollector = structs.NewServiceName(apimod.HCPMetricsCollectorName, nil) + + apiUID = NewUpstreamIDFromServiceName(api) + dbUID = NewUpstreamIDFromServiceName(db) + pqUID = UpstreamIDFromString("prepared_query:query") + extApiUID = NewUpstreamIDFromServiceName(apiA) + extDBUID = NewUpstreamIDFromServiceName(db) + hcpCollectorUID = NewUpstreamIDFromServiceName(hcpCollector) ) // TODO(peering): NewUpstreamIDFromServiceName should take a PeerName extApiUID.Peer = "peer-a" @@ -3623,6 +3626,164 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, }, }, + "hcp-metrics": { + ns: structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "web-sidecar-proxy", + Service: "web-sidecar-proxy", + Address: "10.0.1.1", + Port: 443, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "web", + Config: map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/tmp/consul/hcp-metrics/", + }, + }, + }, + sourceDC: "dc1", + stages: []verificationStage{ + { + requiredWatches: map[string]verifyWatchRequest{ + fmt.Sprintf("discovery-chain:%s", hcpCollectorUID.String()): genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ + Name: hcpCollector.Name, + EvaluateInDatacenter: "dc1", + EvaluateInNamespace: "default", + EvaluateInPartition: "default", + Datacenter: "dc1", + QueryOptions: structs.QueryOptions{ + Token: aclToken, + }, + }), + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.False(t, snap.Valid(), "should not be valid") + + require.Len(t, snap.ConnectProxy.DiscoveryChain, 0, "%+v", snap.ConnectProxy.DiscoveryChain) + require.Len(t, snap.ConnectProxy.WatchedDiscoveryChains, 0, "%+v", snap.ConnectProxy.WatchedDiscoveryChains) + require.Len(t, snap.ConnectProxy.WatchedUpstreams, 0, "%+v", snap.ConnectProxy.WatchedUpstreams) + require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 0, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints) + }, + }, + { + events: []UpdateEvent{ + rootWatchEvent(), + { + CorrelationID: peeringTrustBundlesWatchID, + Result: peerTrustBundles, + }, + { + CorrelationID: leafWatchID, + Result: issuedCert, + Err: nil, + }, + { + CorrelationID: intentionsWatchID, + Result: TestIntentions(), + Err: nil, + }, + { + CorrelationID: meshConfigEntryID, + Result: &structs.ConfigEntryResponse{}, + }, + { + CorrelationID: fmt.Sprintf("discovery-chain:%s", hcpCollectorUID.String()), + Result: &structs.DiscoveryChainResponse{ + Chain: discoverychain.TestCompileConfigEntries(t, hcpCollector.Name, "default", "default", "dc1", "trustdomain.consul", nil), + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.Equal(t, indexedRoots, snap.Roots) + require.Equal(t, issuedCert, snap.ConnectProxy.Leaf) + + // An event was received with the HCP collector's discovery chain, which sets up some bookkeeping in the snapshot. + require.Len(t, snap.ConnectProxy.DiscoveryChain, 1, "%+v", snap.ConnectProxy.DiscoveryChain) + require.Contains(t, snap.ConnectProxy.DiscoveryChain, hcpCollectorUID) + + require.Len(t, snap.ConnectProxy.WatchedUpstreams, 1, "%+v", snap.ConnectProxy.WatchedUpstreams) + require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 1, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints) + require.Contains(t, snap.ConnectProxy.WatchedUpstreamEndpoints, hcpCollectorUID) + + expectUpstream := structs.Upstream{ + DestinationNamespace: "default", + DestinationPartition: "default", + DestinationName: apimod.HCPMetricsCollectorName, + LocalBindSocketPath: "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock", + Config: map[string]interface{}{ + "protocol": "grpc", + }, + } + uid := NewUpstreamID(&expectUpstream) + + require.Contains(t, snap.ConnectProxy.UpstreamConfig, uid) + require.Equal(t, &expectUpstream, snap.ConnectProxy.UpstreamConfig[uid]) + + // No endpoints have arrived yet. + require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints[hcpCollectorUID], 0, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints) + }, + }, + { + requiredWatches: map[string]verifyWatchRequest{ + fmt.Sprintf("upstream-target:%s.default.default.dc1:", apimod.HCPMetricsCollectorName) + hcpCollectorUID.String(): genVerifyServiceSpecificRequest(apimod.HCPMetricsCollectorName, "", "dc1", true), + }, + events: []UpdateEvent{ + { + CorrelationID: fmt.Sprintf("upstream-target:%s.default.default.dc1:", apimod.HCPMetricsCollectorName) + hcpCollectorUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.0.1", + }, + Service: &structs.NodeService{ + ID: apimod.HCPMetricsCollectorName, + Service: apimod.HCPMetricsCollectorName, + Port: 8080, + }, + }, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.Equal(t, indexedRoots, snap.Roots) + require.Equal(t, issuedCert, snap.ConnectProxy.Leaf) + + // Discovery chain for the HCP collector should still be stored in the snapshot. + require.Len(t, snap.ConnectProxy.DiscoveryChain, 1, "%+v", snap.ConnectProxy.DiscoveryChain) + require.Contains(t, snap.ConnectProxy.DiscoveryChain, hcpCollectorUID) + + require.Len(t, snap.ConnectProxy.WatchedUpstreams, 1, "%+v", snap.ConnectProxy.WatchedUpstreams) + require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints, 1, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints) + require.Contains(t, snap.ConnectProxy.WatchedUpstreamEndpoints, hcpCollectorUID) + + // An endpoint arrived for the HCP collector, so it should be present in the snapshot. + require.Len(t, snap.ConnectProxy.WatchedUpstreamEndpoints[hcpCollectorUID], 1, "%+v", snap.ConnectProxy.WatchedUpstreamEndpoints) + + nodes := structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "10.0.0.1", + }, + Service: &structs.NodeService{ + ID: apimod.HCPMetricsCollectorName, + Service: apimod.HCPMetricsCollectorName, + Port: 8080, + }, + }, + } + target := fmt.Sprintf("%s.default.default.dc1", apimod.HCPMetricsCollectorName) + require.Equal(t, nodes, snap.ConnectProxy.WatchedUpstreamEndpoints[hcpCollectorUID][target]) + }, + }, + }, + }, } for name, tc := range cases { diff --git a/agent/proxycfg/testing_connect_proxy.go b/agent/proxycfg/testing_connect_proxy.go index 74ac5cb86709..394687a44bcc 100644 --- a/agent/proxycfg/testing_connect_proxy.go +++ b/agent/proxycfg/testing_connect_proxy.go @@ -1,6 +1,7 @@ package proxycfg import ( + "fmt" "time" "github.com/mitchellh/go-testing-interface" @@ -9,6 +10,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/types" ) @@ -288,3 +290,51 @@ func TestConfigSnapshotGRPCExposeHTTP1(t testing.T) *ConfigSnapshot { }, }) } + +// TestConfigSnapshotDiscoveryChain returns a fully populated snapshot using a discovery chain +func TestConfigSnapshotHCPMetrics(t testing.T) *ConfigSnapshot { + // DiscoveryChain without an UpstreamConfig should yield a + // filter chain when in transparent proxy mode + var ( + collector = structs.NewServiceName(api.HCPMetricsCollectorName, nil) + collectorUID = NewUpstreamIDFromServiceName(collector) + collectorChain = discoverychain.TestCompileConfigEntries(t, api.HCPMetricsCollectorName, "default", "default", "dc1", connect.TestClusterID+".consul", nil) + ) + + return TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Config = map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/tmp/consul/hcp-metrics", + } + }, []UpdateEvent{ + { + CorrelationID: meshConfigEntryID, + Result: &structs.ConfigEntryResponse{ + Entry: nil, + }, + }, + { + CorrelationID: "discovery-chain:" + collectorUID.String(), + Result: &structs.DiscoveryChainResponse{ + Chain: collectorChain, + }, + }, + { + CorrelationID: fmt.Sprintf("upstream-target:%s.default.default.dc1:", api.HCPMetricsCollectorName) + collectorUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: []structs.CheckServiceNode{ + { + Node: &structs.Node{ + Address: "8.8.8.8", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: api.HCPMetricsCollectorName, + Address: "9.9.9.9", + Port: 9090, + }, + }, + }, + }, + }, + }) +} diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 5fc31dc9e2d9..d5009f63685f 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -166,6 +166,10 @@ func TestAllResourcesFromSnapshot(t *testing.T) { name: "local-mesh-gateway-with-peered-upstreams", create: proxycfg.TestConfigSnapshotPeeringLocalMeshGateway, }, + { + name: "hcp-metrics", + create: proxycfg.TestConfigSnapshotHCPMetrics, + }, } tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) diff --git a/agent/xds/testdata/clusters/hcp-metrics.latest.golden b/agent/xds/testdata/clusters/hcp-metrics.latest.golden new file mode 100644 index 000000000000..441763f1a925 --- /dev/null +++ b/agent/xds/testdata/clusters/hcp-metrics.latest.golden @@ -0,0 +1,183 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + }, + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + } + ] + } + }, + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + }, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/hcp-metrics-collector" + } + ] + } + }, + "sni": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/hcp-metrics.latest.golden b/agent/xds/testdata/endpoints/hcp-metrics.latest.golden new file mode 100644 index 000000000000..a19ac95f210f --- /dev/null +++ b/agent/xds/testdata/endpoints/hcp-metrics.latest.golden @@ -0,0 +1,97 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "9.9.9.9", + "portValue": 9090 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/hcp-metrics.latest.golden b/agent/xds/testdata/listeners/hcp-metrics.latest.golden new file mode 100644 index 000000000000..d89036a935cb --- /dev/null +++ b/agent/xds/testdata/listeners/hcp-metrics.latest.golden @@ -0,0 +1,184 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "hcp-metrics-collector:/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock", + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "upstream.hcp-metrics-collector.default.default.dc1", + "routeConfig": { + "name": "hcp-metrics-collector", + "virtualHosts": [ + { + "name": "hcp-metrics-collector.default.default.dc1", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "hcp-metrics-collector.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ] + }, + "httpFilters": [ + { + "name": "envoy.filters.http.grpc_stats", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig", + "statsForAllMethods": true + } + }, + { + "name": "envoy.filters.http.grpc_http1_bridge", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_http1_bridge.v3.Config" + } + }, + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": {} + }, + "http2ProtocolOptions": {} + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": {}, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/hcp-metrics.latest.golden b/agent/xds/testdata/routes/hcp-metrics.latest.golden new file mode 100644 index 000000000000..306f5220e7b9 --- /dev/null +++ b/agent/xds/testdata/routes/hcp-metrics.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/hcp-metrics.latest.golden b/agent/xds/testdata/secrets/hcp-metrics.latest.golden new file mode 100644 index 000000000000..e6c25e165c65 --- /dev/null +++ b/agent/xds/testdata/secrets/hcp-metrics.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/api/connect.go b/api/connect.go index a40d1e2321ab..a5298d81369b 100644 --- a/api/connect.go +++ b/api/connect.go @@ -1,5 +1,8 @@ package api +// HCPMetricsCollectorName is the service name for the HCP Metrics Collector +const HCPMetricsCollectorName string = "hcp-metrics-collector" + // Connect can be used to work with endpoints related to Connect, the // feature for securely connecting services within Consul. type Connect struct { diff --git a/command/connect/envoy/bootstrap_config.go b/command/connect/envoy/bootstrap_config.go index 23427ad0af62..e88d83e6a0dd 100644 --- a/command/connect/envoy/bootstrap_config.go +++ b/command/connect/envoy/bootstrap_config.go @@ -7,9 +7,11 @@ import ( "net" "net/url" "os" + "path" "strings" "text/template" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/api" ) @@ -49,6 +51,11 @@ type BootstrapConfig struct { // stats_config.stats_tags can be made by overriding envoy_stats_config_json. StatsTags []string `mapstructure:"envoy_stats_tags"` + // HCPMetricsBindSocketDir is a string that configures the directory for a + // unix socket where Envoy will forward metrics. These metrics get pushed to + // the HCP Metrics collector to show service mesh metrics on HCP. + HCPMetricsBindSocketDir string `mapstructure:"envoy_hcp_metrics_bind_socket_dir"` + // PrometheusBindAddr configures an : on which the Envoy will listen // and expose a single /metrics HTTP endpoint for Prometheus to scrape. It // does this by proxying that URL to the internal admin server's prometheus @@ -238,6 +245,11 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs, omitDeprecatedTa args.StatsFlushInterval = c.StatsFlushInterval } + // Setup HCP Metrics if needed. This MUST happen after the Static*JSON is set above + if c.HCPMetricsBindSocketDir != "" { + appendHCPMetricsConfig(args, c.HCPMetricsBindSocketDir) + } + return nil } @@ -271,7 +283,7 @@ func (c *BootstrapConfig) generateStatsSinks(args *BootstrapTplArgs) error { } if len(stats_sinks) > 0 { - args.StatsSinksJSON = "[\n" + strings.Join(stats_sinks, ",\n") + "\n]" + args.StatsSinksJSON = strings.Join(stats_sinks, ",\n") } return nil } @@ -796,6 +808,58 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd return nil } +// appendHCPMetricsConfig generates config to enable a socket at path: /_.sock +// or /.sock, if namespace is empty. +func appendHCPMetricsConfig(args *BootstrapTplArgs, hcpMetricsBindSocketDir string) { + // Normalize namespace to "default". This ensures we match the namespace behaviour in proxycfg package, + // where a dynamic listener will be created at the same socket path via xDS. + sock := fmt.Sprintf("%s_%s.sock", acl.NamespaceOrDefault(args.Namespace), args.ProxyID) + path := path.Join(hcpMetricsBindSocketDir, sock) + + if args.StatsSinksJSON != "" { + args.StatsSinksJSON += ",\n" + } + args.StatsSinksJSON += `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }` + + if args.StaticClustersJSON != "" { + args.StaticClustersJSON += ",\n" + } + args.StaticClustersJSON += fmt.Sprintf(`{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "%s" + } + } + } + } + ] + } + ] + } + }`, path) +} + func containsSelfAdminCluster(clustersJSON string) (bool, error) { clusterNames := []struct { Name string diff --git a/command/connect/envoy/bootstrap_config_test.go b/command/connect/envoy/bootstrap_config_test.go index 9e8038ae0365..e5d9548e655c 100644 --- a/command/connect/envoy/bootstrap_config_test.go +++ b/command/connect/envoy/bootstrap_config_test.go @@ -513,6 +513,56 @@ const ( } ] }` + + expectedStatsdSink = `{ + "name": "envoy.stat_sinks.statsd", + "typedConfig": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9125 + } + } + } +}` + + expectedHCPMetricsStatsSink = `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }` + + expectedHCPMetricsCluster = `{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + } + } + } + } + ] + } + ] + } + }` ) func TestBootstrapConfig_ConfigureArgs(t *testing.T) { @@ -557,33 +607,71 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.custom_exciting_sink", "config": { "foo": "bar" } - }]`, + }`, }, }, { - name: "simple-statsd-sink", + name: "hcp-metrics-sink", + baseArgs: BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + }, input: BootstrapConfig{ - StatsdURL: "udp://127.0.0.1:9125", + HCPMetricsBindSocketDir: "/tmp/consul/hcp-metrics", }, wantArgs: BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ - "name": "envoy.stat_sinks.statsd", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", - "address": { - "socket_address": { - "address": "127.0.0.1", - "port_value": 9125 + StatsSinksJSON: `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }`, + StaticClustersJSON: `{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + } + } + } } + ] } + ] } - }]`, + }`, + }, + wantErr: false, + }, + { + name: "simple-statsd-sink", + input: BootstrapConfig{ + StatsdURL: "udp://127.0.0.1:9125", + }, + wantArgs: BootstrapTplArgs{ + StatsConfigJSON: defaultStatsConfigJSON, + StatsSinksJSON: expectedStatsdSink, }, wantErr: false, }, @@ -600,7 +688,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -617,7 +705,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { "config": { "foo": "bar" } - }]`, + }`, }, wantErr: false, }, @@ -629,7 +717,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"MY_STATSD_URL=udp://127.0.0.1:9125"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -640,7 +728,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -652,7 +740,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"HOST_IP=127.0.0.1"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -663,7 +751,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -685,7 +773,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -696,7 +784,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -707,7 +795,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -717,7 +805,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -730,7 +818,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"MY_STATSD_URL=udp://127.0.0.1:9125"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -741,7 +829,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -1539,3 +1627,65 @@ func TestConsulTagSpecifiers(t *testing.T) { }) } } + +func TestAppendHCPMetrics(t *testing.T) { + tests := map[string]struct { + inputArgs *BootstrapTplArgs + bindSocketDir string + wantArgs *BootstrapTplArgs + }{ + "dir-without-trailing-slash": { + inputArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedHCPMetricsStatsSink, + StaticClustersJSON: expectedHCPMetricsCluster, + }, + }, + "dir-with-trailing-slash": { + inputArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedHCPMetricsStatsSink, + StaticClustersJSON: expectedHCPMetricsCluster, + }, + }, + "append-clusters-and-stats-sink": { + inputArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedStatsdSink, + StaticClustersJSON: expectedSelfAdminCluster, + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedStatsdSink + ",\n" + expectedHCPMetricsStatsSink, + StaticClustersJSON: expectedSelfAdminCluster + ",\n" + expectedHCPMetricsCluster, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + appendHCPMetricsConfig(tt.inputArgs, tt.bindSocketDir) + + // Some of our JSON strings are comma separated objects to be + // insertedinto an array which is not valid JSON on it's own so wrap + // them all in an array. For simple values this is still valid JSON + // too. + wantStatsSink := "[" + tt.wantArgs.StatsSinksJSON + "]" + gotStatsSink := "[" + tt.inputArgs.StatsSinksJSON + "]" + require.JSONEq(t, wantStatsSink, gotStatsSink, "field StatsSinksJSON should be equivalent JSON") + + wantClusters := "[" + tt.wantArgs.StaticClustersJSON + "]" + gotClusters := "[" + tt.inputArgs.StaticClustersJSON + "]" + require.JSONEq(t, wantClusters, gotClusters, "field StaticClustersJSON should be equivalent JSON") + }) + } +} diff --git a/command/connect/envoy/bootstrap_tpl.go b/command/connect/envoy/bootstrap_tpl.go index 7ed75304bcad..9e264fe9351b 100644 --- a/command/connect/envoy/bootstrap_tpl.go +++ b/command/connect/envoy/bootstrap_tpl.go @@ -262,7 +262,9 @@ const bootstrapTemplate = `{ {{- end }} }, {{- if .StatsSinksJSON }} - "stats_sinks": {{ .StatsSinksJSON }}, + "stats_sinks": [ + {{ .StatsSinksJSON }} + ], {{- end }} {{- if .StatsConfigJSON }} "stats_config": {{ .StatsConfigJSON }}, diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index e2692c77d7c3..8366d65974ea 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -195,6 +195,29 @@ func TestGenerateConfig(t *testing.T) { PrometheusScrapePath: "/metrics", }, }, + { + Name: "hcp-metrics", + Flags: []string{"-proxy-id", "test-proxy"}, + ProxyConfig: map[string]interface{}{ + "envoy_hcp_metrics_bind_socket_dir": "/tmp/consul/hcp-metrics", + }, + WantArgs: BootstrapTplArgs{ + ProxyCluster: "test-proxy", + ProxyID: "test-proxy", + // We don't know this til after the lookup so it will be empty in the + // initial args call we are testing here. + ProxySourceService: "", + GRPC: GRPC{ + AgentAddress: "127.0.0.1", + AgentPort: "8502", + }, + AdminAccessLogPath: "/dev/null", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", + }, + }, { Name: "prometheus-metrics", Flags: []string{"-proxy-id", "test-proxy", diff --git a/command/connect/envoy/testdata/hcp-metrics.golden b/command/connect/envoy/testdata/hcp-metrics.golden new file mode 100644 index 000000000000..563d662e4436 --- /dev/null +++ b/command/connect/envoy/testdata/hcp-metrics.golden @@ -0,0 +1,247 @@ +{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "test", + "id": "test-proxy", + "metadata": { + "namespace": "default", + "partition": "default" + } + }, + "layered_runtime": { + "layers": [ + { + "name": "base", + "static_layer": { + "re2.max_program_size.error_level": 1048576 + } + } + ] + }, + "static_resources": { + "clusters": [ + { + "name": "local_agent", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "local_agent", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8502 + } + } + } + } + ] + } + ] + } + }, + { + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_test-proxy.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "stats_sinks": [ + { + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + } + ], + "stats_config": { + "stats_tags": [ + { + "regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.custom_hash" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service_subset" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.namespace" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.partition" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.datacenter" + }, + { + "regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.peer" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.routing_type" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.destination.trust_domain" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.target" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.destination.full_target" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.service" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.datacenter" + }, + { + "regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.peer" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.namespace" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)", + "tag_name": "consul.upstream.partition" + }, + { + "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.custom_hash" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service_subset" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.namespace" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.datacenter" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.routing_type" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.trust_domain" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.target" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.full_target" + }, + { + "tag_name": "local_cluster", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.service", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.namespace", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.partition", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.datacenter", + "fixed_value": "dc1" + } + ], + "use_all_default_tags": true + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "DELTA_GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "" + } + ], + "envoy_grpc": { + "cluster_name": "local_agent" + } + } + } + } +} + diff --git a/website/content/commands/connect/envoy.mdx b/website/content/commands/connect/envoy.mdx index 90bccf2faf8c..c35a0b4feaa8 100644 --- a/website/content/commands/connect/envoy.mdx +++ b/website/content/commands/connect/envoy.mdx @@ -75,6 +75,10 @@ Usage: `consul connect envoy [options] [-- pass-through options]` In cases where either assumption is violated this flag will prevent the command attempting to resolve config from the local agent. +- `envoy_hcp_metrics_bind_socket_dir` - Specifies the directory where Envoy creates a unix socket. + Envoy sends metrics to the socket so that HCP collectors can connect to collect them." + The socket is not configured by default. + - `-envoy-ready-bind-address` - By default the proxy does not have a readiness probe configured on it. This flag in conjunction with the `envoy-ready-bind-port` flag configures where the envoy readiness probe is configured on the proxy. A `/ready` HTTP From 04d882f9830465a0c7212cce8629fa46f0082c70 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 13 Mar 2023 14:54:40 -0700 Subject: [PATCH 105/421] Backport of Preserve CARoots when updating Vault CA configuration into release/1.15.x (#16626) * backport of commit 537734d2ec18092718d2c297cc7cf8a7d9818314 * backport of commit 523d3136719db8e304eb85409fca7cfd0d49da09 * backport of commit 8a113841d47c0757d632790e4e05161aa7004899 * backport of commit 368f8a51e98a5aab4b11d580588bea3d25315ed4 --------- Co-authored-by: Chris S. Kim --- .changelog/16592.txt | 3 + agent/consul/leader_connect_ca.go | 37 +++++++----- agent/consul/leader_connect_ca_test.go | 81 ++++++++++++++++++-------- 3 files changed, 82 insertions(+), 39 deletions(-) create mode 100644 .changelog/16592.txt diff --git a/.changelog/16592.txt b/.changelog/16592.txt new file mode 100644 index 000000000000..ba37d69015ff --- /dev/null +++ b/.changelog/16592.txt @@ -0,0 +1,3 @@ +```release-note:bug +ca: Fixes a bug where updating Vault CA Provider config would cause TLS issues in the service mesh +``` diff --git a/agent/consul/leader_connect_ca.go b/agent/consul/leader_connect_ca.go index 008289c947e6..abb92f54b6bb 100644 --- a/agent/consul/leader_connect_ca.go +++ b/agent/consul/leader_connect_ca.go @@ -12,7 +12,7 @@ import ( "time" "github.com/hashicorp/go-hclog" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-uuid" "golang.org/x/time/rate" "github.com/hashicorp/consul/acl" @@ -272,7 +272,7 @@ func newCARoot(pemValue, provider, clusterID string) (*structs.CARoot, error) { ExternalTrustDomain: clusterID, NotBefore: primaryCert.NotBefore, NotAfter: primaryCert.NotAfter, - RootCert: pemValue, + RootCert: lib.EnsureTrailingNewline(pemValue), PrivateKeyType: keyType, PrivateKeyBits: keyBits, Active: true, @@ -887,6 +887,23 @@ func (c *CAManager) primaryUpdateRootCA(newProvider ca.Provider, args *structs.C return err } + // TODO: https://github.com/hashicorp/consul/issues/12386 + intermediate, err := newProvider.ActiveIntermediate() + if err != nil { + return fmt.Errorf("error fetching active intermediate: %w", err) + } + if intermediate == "" { + intermediate, err = newProvider.GenerateIntermediate() + if err != nil { + return fmt.Errorf("error generating intermediate: %w", err) + } + } + if intermediate != newRootPEM { + if err := setLeafSigningCert(newActiveRoot, intermediate); err != nil { + return err + } + } + // See if the provider needs to persist any state along with the config pState, err := newProvider.State() if err != nil { @@ -970,19 +987,9 @@ func (c *CAManager) primaryUpdateRootCA(newProvider ca.Provider, args *structs.C } // Add the cross signed cert to the new CA's intermediates (to be attached - // to leaf certs). - newActiveRoot.IntermediateCerts = []string{xcCert} - } - } - - // TODO: https://github.com/hashicorp/consul/issues/12386 - intermediate, err := newProvider.GenerateIntermediate() - if err != nil { - return err - } - if intermediate != newRootPEM { - if err := setLeafSigningCert(newActiveRoot, intermediate); err != nil { - return err + // to leaf certs). We do not want it to be the last cert if there are any + // existing intermediate certs so we push to the front. + newActiveRoot.IntermediateCerts = append([]string{xcCert}, newActiveRoot.IntermediateCerts...) } } diff --git a/agent/consul/leader_connect_ca_test.go b/agent/consul/leader_connect_ca_test.go index 7e84a87b19bf..1e4e4d2af960 100644 --- a/agent/consul/leader_connect_ca_test.go +++ b/agent/consul/leader_connect_ca_test.go @@ -25,7 +25,7 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" - ca "github.com/hashicorp/consul/agent/connect/ca" + "github.com/hashicorp/consul/agent/connect/ca" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" @@ -612,39 +612,72 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) { _, origRoot, err := s1.fsm.State().CARootActive(nil) require.NoError(t, err) require.Len(t, origRoot.IntermediateCerts, 1) + origRoot.CreateIndex = s1.caManager.providerRoot.CreateIndex + origRoot.ModifyIndex = s1.caManager.providerRoot.ModifyIndex + require.Equal(t, s1.caManager.providerRoot, origRoot) cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(origRoot)) require.NoError(t, err) require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID) - vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{ - RootPath: "pki-root-2", - IntermediatePath: "pki-intermediate-2", - ConsulManaged: true, + t.Run("update config without changing root", func(t *testing.T) { + err = s1.caManager.UpdateConfiguration(&structs.CARequest{ + Config: &structs.CAConfiguration{ + Provider: "vault", + Config: map[string]interface{}{ + "Address": vault.Addr, + "Token": vaultToken, + "RootPKIPath": "pki-root/", + "IntermediatePKIPath": "pki-intermediate/", + "CSRMaxPerSecond": 100, + }, + }, + }) + require.NoError(t, err) + _, sameRoot, err := s1.fsm.State().CARootActive(nil) + require.NoError(t, err) + require.Len(t, sameRoot.IntermediateCerts, 1) + sameRoot.CreateIndex = s1.caManager.providerRoot.CreateIndex + sameRoot.ModifyIndex = s1.caManager.providerRoot.ModifyIndex + + cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(sameRoot)) + require.NoError(t, err) + require.Equal(t, connect.HexString(cert.SubjectKeyId), sameRoot.SigningKeyID) + + require.Equal(t, origRoot, sameRoot) + require.Equal(t, sameRoot, s1.caManager.providerRoot) }) - err = s1.caManager.UpdateConfiguration(&structs.CARequest{ - Config: &structs.CAConfiguration{ - Provider: "vault", - Config: map[string]interface{}{ - "Address": vault.Addr, - "Token": vaultToken2, - "RootPKIPath": "pki-root-2/", - "IntermediatePKIPath": "pki-intermediate-2/", + t.Run("update config and change root", func(t *testing.T) { + vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{ + RootPath: "pki-root-2", + IntermediatePath: "pki-intermediate-2", + ConsulManaged: true, + }) + + err = s1.caManager.UpdateConfiguration(&structs.CARequest{ + Config: &structs.CAConfiguration{ + Provider: "vault", + Config: map[string]interface{}{ + "Address": vault.Addr, + "Token": vaultToken2, + "RootPKIPath": "pki-root-2/", + "IntermediatePKIPath": "pki-intermediate-2/", + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - _, newRoot, err := s1.fsm.State().CARootActive(nil) - require.NoError(t, err) - require.Len(t, newRoot.IntermediateCerts, 2, - "expected one cross-sign cert and one local leaf sign cert") - require.NotEqual(t, origRoot.ID, newRoot.ID) + _, newRoot, err := s1.fsm.State().CARootActive(nil) + require.NoError(t, err) + require.Len(t, newRoot.IntermediateCerts, 2, + "expected one cross-sign cert and one local leaf sign cert") + require.NotEqual(t, origRoot.ID, newRoot.ID) - cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot)) - require.NoError(t, err) - require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID) + cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot)) + require.NoError(t, err) + require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID) + }) } func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) { From 572ec685a8eade8cc534464b39ed4f8c3f67861e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Mar 2023 04:57:57 -0700 Subject: [PATCH 106/421] Backport of Add known issues to Raft WAL docs. into release/1.15.x (#16638) * backport of commit 00ca421957b57354e6b1c9fa6fde9e699ec1649e * backport of commit 2e3c67c773e1cc5aea93cc568bb4fbce339a6a57 --------- Co-authored-by: Paul Banks --- .../content/docs/agent/wal-logstore/enable.mdx | 15 ++++++++++++--- website/content/docs/agent/wal-logstore/index.mdx | 13 +++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/website/content/docs/agent/wal-logstore/enable.mdx b/website/content/docs/agent/wal-logstore/enable.mdx index a4a89f70c86e..cdd9b933dfd5 100644 --- a/website/content/docs/agent/wal-logstore/enable.mdx +++ b/website/content/docs/agent/wal-logstore/enable.mdx @@ -7,7 +7,7 @@ description: >- # Enable the experimental WAL LogStore backend -This topic describes how to safely configure and test the WAL backend in your Consul deployment. +This topic describes how to safely configure and test the WAL backend in your Consul deployment. The overall process for enabling the WAL LogStore backend for one server consists of the following steps. In production environments, we recommend starting by enabling the backend on a single server . If you eventually choose to expand the test to further servers, you must repeat these steps for each one. @@ -17,9 +17,9 @@ The overall process for enabling the WAL LogStore backend for one server consist 1. Remove data directory from target server. 1. Update target server's configuration. 1. Start the target server. -1. Monitor target server raft metrics and logs. +1. Monitor target server raft metrics and logs. -!> **Experimental feature:** The WAL LogStore backend is experimental. +!> **Experimental feature:** The WAL LogStore backend is experimental and may contain bugs that could cause data loss. Follow this guide to manage risk during testing. ## Requirements @@ -32,6 +32,15 @@ We recommend taking the following additional measures: - Monitor Consul server metrics and logs, and set an alert on specific log events that occur when WAL is enabled. Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for more information. - Enable WAL in a pre-production environment and run it for a several days before enabling it in production. +## Known issues + +The following issues were discovered after release of Consul 1.15.1 and will be +fixed in a future patch release. + + * A follower that is disconnected may be unable to catch up if it is using the WAL backend. + * Restoring user snapshots can break replication to WAL-enabled followers. + * Restoring user snapshots can cause a WAL-enabled leader to panic. + ## Risks While their likelihood remains low to very low, be aware of the following risks before implementing the WAL backend: diff --git a/website/content/docs/agent/wal-logstore/index.mdx b/website/content/docs/agent/wal-logstore/index.mdx index 914d6602a9e9..491255e8ed5a 100644 --- a/website/content/docs/agent/wal-logstore/index.mdx +++ b/website/content/docs/agent/wal-logstore/index.mdx @@ -7,17 +7,22 @@ description: >- # Experimental WAL LogStore backend overview -This topic provides an overview of the experimental WAL (write-ahead log) LogStore backend. +This topic provides an overview of the WAL (write-ahead log) LogStore backend. +The WAL backend is an experimental feature. Refer to +[Requirements](/consul/docs/agent/wal-logstore/enable#requirements) for +supported environments and known issues. -!> **Experimental feature:** The WAL LogStore backend is experimental. +We do not recommend enabling the WAL backend in production without following +[our guide for safe +testing](/consul/docs/agent/wal-logstore/enable). ## WAL versus BoltDB -WAL implements a traditional log with rotating, append-only log files. WAL resolves many issues with the existing `LogStore` provided by the BoltDB backend. The BoltDB `LogStore` is a copy-on-write BTree, which is not optimized for append-only, write-heavy workloads. +WAL implements a traditional log with rotating, append-only log files. WAL resolves many issues with the existing `LogStore` provided by the BoltDB backend. The BoltDB `LogStore` is a copy-on-write BTree, which is not optimized for append-only, write-heavy workloads. ### BoltDB storage scalability issues -The existing BoltDB log store inefficiently stores append-only logs to disk because it was designed as a full key-value database. It is a single file that only ever grows. Deleting the oldest logs, which Consul does regularly when it makes new snapshots of the state, leaves free space in the file. The free space must be tracked in a `freelist` so that BoltDB can reuse it on future writes. By contrast, a simple segmented log can delete the oldest log files from disk. +The existing BoltDB log store inefficiently stores append-only logs to disk because it was designed as a full key-value database. It is a single file that only ever grows. Deleting the oldest logs, which Consul does regularly when it makes new snapshots of the state, leaves free space in the file. The free space must be tracked in a `freelist` so that BoltDB can reuse it on future writes. By contrast, a simple segmented log can delete the oldest log files from disk. A burst of writes at double or triple the normal volume can suddenly cause the log file to grow to several times its steady-state size. After Consul takes the next snapshot and truncates the oldest logs, the resulting file is mostly empty space. From 89ce3ba12310c52d937b4f5973d8e636fbb2d2af Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 15 Mar 2023 06:08:14 -0700 Subject: [PATCH 107/421] backport of commit 78bb205fc3c788aee4fbfd4444cd4075ecd43f07 (#16635) Co-authored-by: Derek Menteer --- .changelog/_4696.txt | 3 +++ agent/proxycfg-glue/trust_bundle.go | 6 ++++-- agent/rpc/peering/service.go | 9 ++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 .changelog/_4696.txt diff --git a/.changelog/_4696.txt b/.changelog/_4696.txt new file mode 100644 index 000000000000..951896fd66f3 --- /dev/null +++ b/.changelog/_4696.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: **(Consul Enterprise only)** Fix issue where connect-enabled services with peer upstreams incorrectly required `service:write` access in the `default` namespace to query data, which was too restrictive. Now having `service:write` to any namespace is sufficient to query the peering data. +``` diff --git a/agent/proxycfg-glue/trust_bundle.go b/agent/proxycfg-glue/trust_bundle.go index 4d6cbc4d2393..05d2619fa693 100644 --- a/agent/proxycfg-glue/trust_bundle.go +++ b/agent/proxycfg-glue/trust_bundle.go @@ -33,12 +33,14 @@ type serverTrustBundle struct { } func (s *serverTrustBundle) Notify(ctx context.Context, req *cachetype.TrustBundleReadRequest, correlationID string, ch chan<- proxycfg.UpdateEvent) error { - entMeta := structs.NodeEnterpriseMetaInPartition(req.Request.Partition) + // Having the ability to write a service in ANY (at least one) namespace should be + // sufficient for reading the trust bundle, which is why we use a wildcard. + entMeta := acl.NewEnterpriseMetaWithPartition(req.Request.Partition, acl.WildcardName) return watch.ServerLocalNotify(ctx, correlationID, s.deps.GetStore, func(ws memdb.WatchSet, store Store) (uint64, *pbpeering.TrustBundleReadResponse, error) { var authzCtx acl.AuthorizerContext - authz, err := s.deps.ACLResolver.ResolveTokenAndDefaultMeta(req.Token, entMeta, &authzCtx) + authz, err := s.deps.ACLResolver.ResolveTokenAndDefaultMeta(req.Token, &entMeta, &authzCtx) if err != nil { return 0, nil, err } diff --git a/agent/rpc/peering/service.go b/agent/rpc/peering/service.go index 9a6b1adaa5d2..e1a289bed5bb 100644 --- a/agent/rpc/peering/service.go +++ b/agent/rpc/peering/service.go @@ -920,9 +920,12 @@ func (s *Server) TrustBundleRead(ctx context.Context, req *pbpeering.TrustBundle defer metrics.MeasureSince([]string{"peering", "trust_bundle_read"}, time.Now()) + // Having the ability to write a service in ANY (at least one) namespace should be + // sufficient for reading the trust bundle, which is why we use a wildcard. + entMeta := acl.NewEnterpriseMetaWithPartition(req.Partition, acl.WildcardName) + entMeta.Normalize() var authzCtx acl.AuthorizerContext - entMeta := structs.DefaultEnterpriseMetaInPartition(req.Partition) - authz, err := s.Backend.ResolveTokenAndDefaultMeta(options.Token, entMeta, &authzCtx) + authz, err := s.Backend.ResolveTokenAndDefaultMeta(options.Token, &entMeta, &authzCtx) if err != nil { return nil, err } @@ -933,7 +936,7 @@ func (s *Server) TrustBundleRead(ctx context.Context, req *pbpeering.TrustBundle idx, trustBundle, err := s.Backend.Store().PeeringTrustBundleRead(nil, state.Query{ Value: req.Name, - EnterpriseMeta: *entMeta, + EnterpriseMeta: entMeta, }) if err != nil { return nil, fmt.Errorf("failed to read trust bundle for peer %s: %w", req.Name, err) From 11f14d9ec8152cb6c121e04dd775fa648cbb5de6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 16 Mar 2023 11:05:13 -0700 Subject: [PATCH 108/421] Backport of First cluster grpc service should be NodePort for the second cluster to connect into release/1.15.x (#16653) * backport of commit 9dc24ffd1bf307c40cea44cd3bc294939a1532a7 * backport of commit 2fa4ae4d6af084c2ce49316ccadf215a455a90b9 * backport of commit 7cccb2e1fe058a5d05f62be59452d4405963120b * backport of commit 1442c12573c4473590f9225ceb29d89edf4c2fa1 --------- Co-authored-by: Vipin John Wilson <37441623+vjwilson1987@users.noreply.github.com> --- .../single-dc-multi-k8s.mdx | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx b/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx index b9694b391250..46fc2228016a 100644 --- a/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx +++ b/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx @@ -50,6 +50,17 @@ global: gossipEncryption: secretName: consul-gossip-encryption-key secretKey: key +server: + exposeService: + enabled: true + type: NodePort + nodePort: + ## all are random nodePorts and you can set your own + http: 30010 + https: 30011 + serf: 30012 + rpc: 30013 + grpc: 30014 ui: service: type: NodePort @@ -65,6 +76,8 @@ The UI's service type is set to be `NodePort`. This is needed to connect to servers from another cluster without using the pod IPs of the servers, which are likely going to change. +Other services are exposed as `NodePort` services and configured with random port numbers. In this example, the `grpc` port is set to `30014`, which enables services to discover Consul servers using gRPC when connecting from another cluster. + To deploy, first generate the Gossip encryption key and save it as a Kubernetes secret. ```shell-session @@ -123,6 +136,8 @@ externalServers: hosts: ["10.0.0.4"] # The node port of the UI's NodePort service or the load balancer port. httpsPort: 31557 + # Matches the gRPC port of the Consul servers in the first cluster. + grpcPort: 30014 tlsServerName: server.dc1.consul # The address of the kube API server of this Kubernetes cluster k8sAuthMethodHost: https://kubernetes.example.com:443 @@ -147,6 +162,8 @@ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE cluster1-consul-ui NodePort 10.0.240.80 443:31557/TCP 40h ``` +The `grpcPort: 30014` configuration refers to the gRPC port number specified in the `NodePort` configuration in the first cluster. + Set the `externalServer.tlsServerName` to `server.dc1.consul`. This the DNS SAN (Subject Alternative Name) that is present in the Consul server's certificate. This is required because the connection to the Consul servers uses the node IP, From 283aa59bc4a5b76b1bc7a467d99d447e7a4d01f2 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 16 Mar 2023 16:21:42 -0700 Subject: [PATCH 109/421] Backport of UI: Fix htmlsafe errors throughout the app into release/1.15.x (#16591) * backport of commit abf2894e73ff2f42917cce522a5faa0b6b653439 * backport of commit a8e25402f947af863b94edd9a1d8f1aa3711371e * backport of commit 4dca4afe525ee963edf399ab87647d8cc9434609 --------- Co-authored-by: wenincode --- .changelog/16574.txt | 3 +++ ui/packages/consul-ui/package.json | 2 +- ui/yarn.lock | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 .changelog/16574.txt diff --git a/.changelog/16574.txt b/.changelog/16574.txt new file mode 100644 index 000000000000..78bfc334984f --- /dev/null +++ b/.changelog/16574.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: fix rendering issues on Overview and empty-states by addressing isHTMLSafe errors +``` diff --git a/ui/packages/consul-ui/package.json b/ui/packages/consul-ui/package.json index 445d12235dfc..5898c888faf8 100644 --- a/ui/packages/consul-ui/package.json +++ b/ui/packages/consul-ui/package.json @@ -134,7 +134,7 @@ "ember-export-application-global": "^2.0.1", "ember-in-viewport": "^3.8.1", "ember-inflector": "^4.0.1", - "ember-intl": "^5.5.1", + "ember-intl": "^5.7.0", "ember-load-initializers": "^2.1.2", "ember-math-helpers": "^2.4.0", "ember-maybe-import-regenerator": "^0.1.6", diff --git a/ui/yarn.lock b/ui/yarn.lock index 6eee746a73ea..19ff697e44f5 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -9041,10 +9041,10 @@ ember-inflector@^4.0.2: dependencies: ember-cli-babel "^7.26.5" -ember-intl@^5.5.1: - version "5.6.2" - resolved "https://registry.yarnpkg.com/ember-intl/-/ember-intl-5.6.2.tgz#ece4820923dfda033c279b7e3920cbbc8b6bde07" - integrity sha512-+FfI2udVbnEzueompcRb3ytBWhfnBfVVjAwnCuxwqIyS9ti8lK0ZiYHa5bquNPHjjfBzfFl4x5TlVgDNaCnccg== +ember-intl@^5.7.0: + version "5.7.2" + resolved "https://registry.yarnpkg.com/ember-intl/-/ember-intl-5.7.2.tgz#76d933f974f041448b01247888bc3bcc9261e812" + integrity sha512-gs17uY1ywzMaUpx1gxfBkFQYRTWTSa/zbkL13MVtffG9aBLP+998MibytZOUxIipMtLCm4sr/g6/1aaKRr9/+g== dependencies: broccoli-caching-writer "^3.0.3" broccoli-funnel "^3.0.3" From bc111cb215721efd3cf666c75f2d38ac62f4f551 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 16 Mar 2023 19:12:58 -0700 Subject: [PATCH 110/421] Backport of fix: add AccessorID property to PUT token request into release/1.15.x (#16662) * backport of commit d2c5a64857b73f8032119dd6cdd315f6adf80b92 * backport of commit 8e5e39c25a259c335082569b4ff3210199636a9f --------- Co-authored-by: valeriia-ruban --- .changelog/16660.txt | 3 +++ ui/packages/consul-ui/app/adapters/token.js | 1 + 2 files changed, 4 insertions(+) create mode 100644 .changelog/16660.txt diff --git a/.changelog/16660.txt b/.changelog/16660.txt new file mode 100644 index 000000000000..f8971862165c --- /dev/null +++ b/.changelog/16660.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: fix PUT token request with adding missed AccessorID property to requestBody +``` \ No newline at end of file diff --git a/ui/packages/consul-ui/app/adapters/token.js b/ui/packages/consul-ui/app/adapters/token.js index 5502dbd8994e..4a9f2fa03958 100644 --- a/ui/packages/consul-ui/app/adapters/token.js +++ b/ui/packages/consul-ui/app/adapters/token.js @@ -80,6 +80,7 @@ export default class TokenAdapter extends Adapter { ${{ Description: serialized.Description, + AccessorID: serialized.AccessorID, Policies: serialized.Policies, Roles: serialized.Roles, ServiceIdentities: serialized.ServiceIdentities, From eb63d46abbe647ce249122bc337fa3e733aaf871 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 17 Mar 2023 12:02:28 -0700 Subject: [PATCH 111/421] Backport of [API Gateway] Fix invalid cluster causing gateway programming delay into release/1.15.x (#16668) * backport of commit 9ea73b3b8dfe280416b938c394c8c07f25020305 * backport of commit d3cffdeb4d737a54249808dfd3e526fda2764c9b * backport of commit 0848aac01738c2700965be7c9aa3e7200ca8da88 * backport of commit 90b5e39d2da7140354649bd6e6419a1df4f53ff8 * Refactor and fix flaky tests * Fix bad merge * add file that was never backported * Fix bad merge again * fix duplicate method * remove extra import * backport a slew of testing library code * backport changes coinciding with library update * backport changes coinciding with library update --------- Co-authored-by: Andrew Stucki --- .changelog/16661.txt | 3 + agent/consul/discoverychain/gateway.go | 23 + agent/consul/discoverychain/gateway_test.go | 48 +- agent/xds/resources_test.go | 55 ++- ...route-and-inline-certificate.latest.golden | 55 +++ ...route-and-inline-certificate.latest.golden | 41 ++ ...route-and-inline-certificate.latest.golden | 49 ++ ...route-and-inline-certificate.latest.golden | 31 ++ ...route-and-inline-certificate.latest.golden | 5 + .../consul-container/libs/assert/service.go | 18 +- .../consul-container/libs/cluster/agent.go | 1 + .../consul-container/libs/cluster/builder.go | 5 + .../consul-container/libs/cluster/cluster.go | 46 +- .../libs/cluster/container.go | 9 +- .../consul-container/libs/service/connect.go | 19 +- .../consul-container/libs/service/gateway.go | 18 +- .../consul-container/libs/service/helpers.go | 90 +++- .../libs/topology/peering_topology.go | 181 +++---- .../libs/topology/service_topology.go | 6 +- .../test/basic/connect_service_test.go | 8 +- .../test/gateways/gateway_endpoint_test.go | 290 ++++++----- .../test/gateways/http_route_test.go | 461 +++++++++++------- .../test/gateways/namespace_oss.go | 8 + .../test/observability/access_logs_test.go | 15 +- .../rotate_server_and_ca_then_fail_test.go | 6 +- .../troubleshoot_upstream_test.go | 11 +- .../test/upgrade/acl_node_test.go | 15 +- .../resolver_default_subset_test.go | 23 +- .../upgrade/peering_control_plane_mgw_test.go | 2 +- .../test/upgrade/peering_http_test.go | 24 +- .../test/wanfed/wanfed_peering_test.go | 6 +- 31 files changed, 1022 insertions(+), 550 deletions(-) create mode 100644 .changelog/16661.txt create mode 100644 agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden create mode 100644 agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden create mode 100644 test/integration/consul-container/test/gateways/namespace_oss.go diff --git a/.changelog/16661.txt b/.changelog/16661.txt new file mode 100644 index 000000000000..41336502116e --- /dev/null +++ b/.changelog/16661.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS. +``` diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index 35a8992d80f1..e834b7a1d497 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -128,6 +128,29 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover return nil, nil, err } + node := compiled.Nodes[compiled.StartNode] + if node.IsRouter() { + resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + node.Name + + // clean out the clusters that will get added for the router + for name := range compiled.Nodes { + if strings.HasPrefix(name, resolverPrefix) { + delete(compiled.Nodes, name) + } + } + + // clean out the route rules that'll get added for the router + filtered := []*structs.DiscoveryRoute{} + for _, route := range node.Routes { + if strings.HasPrefix(route.NextNode, resolverPrefix) { + continue + } + filtered = append(filtered, route) + } + node.Routes = filtered + } + compiled.Nodes[compiled.StartNode] = node + // fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present for name, node := range compiled.Nodes { switch node.Type { diff --git a/agent/consul/discoverychain/gateway_test.go b/agent/consul/discoverychain/gateway_test.go index 71e66b051275..57d236afdc81 100644 --- a/agent/consul/discoverychain/gateway_test.go +++ b/agent/consul/discoverychain/gateway_test.go @@ -47,7 +47,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) { route structs.HTTPRouteConfigEntry expectedMatchesByHostname map[string][]hostnameMatch }{ - "no hostanames": { + "no hostnames": { route: structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, Name: "route", @@ -539,15 +539,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { Protocol: "http", StartNode: "router:gateway-suffix-9b9265b.default.default", Nodes: map[string]*structs.DiscoveryGraphNode{ - "resolver:gateway-suffix-9b9265b.default.default.dc1": { - Type: "resolver", - Name: "gateway-suffix-9b9265b.default.default.dc1", - Resolver: &structs.DiscoveryResolver{ - Target: "gateway-suffix-9b9265b.default.default.dc1", - Default: true, - ConnectTimeout: 5000000000, - }, - }, "router:gateway-suffix-9b9265b.default.default": { Type: "router", Name: "gateway-suffix-9b9265b.default.default", @@ -569,20 +560,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }, }, NextNode: "resolver:foo.default.default.dc1", - }, { - Definition: &structs.ServiceRoute{ - Match: &structs.ServiceRouteMatch{ - HTTP: &structs.ServiceRouteHTTPMatch{ - PathPrefix: "/", - }, - }, - Destination: &structs.ServiceRouteDestination{ - Service: "gateway-suffix-9b9265b", - Partition: "default", - Namespace: "default", - }, - }, - NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", }}, }, "resolver:foo.default.default.dc1": { @@ -704,15 +681,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { Protocol: "http", StartNode: "router:gateway-suffix-9b9265b.default.default", Nodes: map[string]*structs.DiscoveryGraphNode{ - "resolver:gateway-suffix-9b9265b.default.default.dc1": { - Type: "resolver", - Name: "gateway-suffix-9b9265b.default.default.dc1", - Resolver: &structs.DiscoveryResolver{ - Target: "gateway-suffix-9b9265b.default.default.dc1", - Default: true, - ConnectTimeout: 5000000000, - }, - }, "resolver:service-one.default.default.dc1": { Type: "resolver", Name: "service-one.default.default.dc1", @@ -770,20 +738,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { }, }, NextNode: "splitter:splitter-one.default.default", - }, { - Definition: &structs.ServiceRoute{ - Match: &structs.ServiceRouteMatch{ - HTTP: &structs.ServiceRouteHTTPMatch{ - PathPrefix: "/", - }, - }, - Destination: &structs.ServiceRouteDestination{ - Service: "gateway-suffix-9b9265b", - Partition: "default", - Namespace: "default", - }, - }, - NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", }}, }, "splitter:splitter-one.default.default": { diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index d5009f63685f..24bee7660615 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -11,6 +11,8 @@ import ( envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -175,7 +177,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) tests = append(tests, getTrafficControlPeeringGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...) - tests = append(tests, getAPIGatewayGoldenTestCases()...) + tests = append(tests, getAPIGatewayGoldenTestCases(t)...) latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { @@ -314,7 +316,13 @@ AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r -----END CERTIFICATE-----` ) -func getAPIGatewayGoldenTestCases() []goldenTestCase { +func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { + t.Helper() + + service := structs.NewServiceName("service", nil) + serviceUID := proxycfg.NewUpstreamIDFromServiceName(service) + serviceChain := discoverychain.TestCompileConfigEntries(t, "service", "default", "default", "dc1", connect.TestClusterID+".consul", nil) + return []goldenTestCase{ { name: "api-gateway-with-tcp-route-and-inline-certificate", @@ -362,5 +370,48 @@ func getAPIGatewayGoldenTestCases() []goldenTestCase { }}, nil) }, }, + { + name: "api-gateway-with-http-route-and-inline-certificate", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolHTTP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Routes: []structs.ResourceReference{{ + Kind: structs.HTTPRoute, + Name: "route", + }}, + }, + } + }, []structs.BoundRoute{ + &structs.HTTPRouteConfigEntry{ + Kind: structs.HTTPRoute, + Name: "route", + Rules: []structs.HTTPRouteRule{{ + Services: []structs.HTTPService{{ + Name: "service", + }}, + }}, + }, + }, nil, []proxycfg.UpdateEvent{{ + CorrelationID: "discovery-chain:" + serviceUID.String(), + Result: &structs.DiscoveryChainResponse{ + Chain: serviceChain, + }, + }, { + CorrelationID: "upstream-target:" + serviceChain.ID() + ":" + serviceUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: proxycfg.TestUpstreamNodes(t, "service"), + }, + }}) + }, + }, } } diff --git a/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..e20479dfd1cf --- /dev/null +++ b/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/service" + } + ] + } + }, + "sni": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..18adccd10c83 --- /dev/null +++ b/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,41 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..e9bee988de93 --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,49 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_8080", + "rds": { + "configSource": { + "ads": {}, + "resourceApiVersion": "V3" + }, + "routeConfigName": "8080" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": {} + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..6abc6f2946b4 --- /dev/null +++ b/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,31 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "8080", + "virtualHosts": [ + { + "name": "api-gateway-listener-9b9265b", + "domains": [ + "*", + "*:8080" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 000000000000..95612291de70 --- /dev/null +++ b/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/consul-container/libs/assert/service.go b/test/integration/consul-container/libs/assert/service.go index 15a03be3b61a..d12c3eb0fdad 100644 --- a/test/integration/consul-container/libs/assert/service.go +++ b/test/integration/consul-container/libs/assert/service.go @@ -24,9 +24,9 @@ const ( ) // CatalogServiceExists verifies the service name exists in the Consul catalog -func CatalogServiceExists(t *testing.T, c *api.Client, svc string) { +func CatalogServiceExists(t *testing.T, c *api.Client, svc string, opts *api.QueryOptions) { retry.Run(t, func(r *retry.R) { - services, _, err := c.Catalog().Service(svc, "", nil) + services, _, err := c.Catalog().Service(svc, "", opts) if err != nil { r.Fatal("error reading service data") } @@ -122,22 +122,22 @@ func ServiceLogContains(t *testing.T, service libservice.Service, target string) // has a `FORTIO_NAME` env variable set. This validates that the client is sending // traffic to the right envoy proxy. // +// If reqHost is set, the Host field of the HTTP request will be set to its value. +// // It retries with timeout defaultHTTPTimeout and wait defaultHTTPWait. -func AssertFortioName(t *testing.T, urlbase string, name string) { +func AssertFortioName(t *testing.T, urlbase string, name string, reqHost string) { t.Helper() var fortioNameRE = regexp.MustCompile(("\nFORTIO_NAME=(.+)\n")) - client := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - }, - } + client := cleanhttp.DefaultClient() retry.RunWith(&retry.Timer{Timeout: defaultHTTPTimeout, Wait: defaultHTTPWait}, t, func(r *retry.R) { fullurl := fmt.Sprintf("%s/debug?env=dump", urlbase) - t.Logf("making call to %s", fullurl) req, err := http.NewRequest("GET", fullurl, nil) if err != nil { r.Fatal("could not make request to service ", fullurl) } + if reqHost != "" { + req.Host = reqHost + } resp, err := client.Do(req) if err != nil { diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 3c4eeafc13ce..8dfa496d8bff 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -17,6 +17,7 @@ type Agent interface { NewClient(string, bool) (*api.Client, error) GetName() string GetAgentName() string + GetPartition() string GetPod() testcontainers.Container ClaimAdminPort() (int, error) GetConfig() Config diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index f08b727d04b1..9ce5bf5c8c34 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -245,6 +245,11 @@ func (b *Builder) Peering(enable bool) *Builder { return b } +func (b *Builder) Partition(name string) *Builder { + b.conf.Set("partition", name) + return b +} + func (b *Builder) RetryJoin(names ...string) *Builder { b.conf.Set("retry_join", names) return b diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index ba24b798c76d..9f80ee573fed 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -66,7 +66,7 @@ func NewN(t TestingT, conf Config, count int) (*Cluster, error) { func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) { id, err := shortid.Generate() if err != nil { - return nil, fmt.Errorf("could not cluster id: %w", err) + return nil, fmt.Errorf("could not generate cluster id: %w", err) } name := fmt.Sprintf("consul-int-cluster-%s", id) @@ -114,7 +114,7 @@ func (c *Cluster) AddN(conf Config, count int, join bool) error { return c.Add(configs, join) } -// Add starts an agent with the given configuration and joins it with the existing cluster +// Add starts agents with the given configurations and joins them to the existing cluster func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) { if c.Index == 0 && !serfJoin { return fmt.Errorf("the first call to Cluster.Add must have serfJoin=true") @@ -125,10 +125,10 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) // Each agent gets it's own area in the cluster scratch. conf.ScratchDir = filepath.Join(c.ScratchDir, strconv.Itoa(c.Index)) if err := os.MkdirAll(conf.ScratchDir, 0777); err != nil { - return err + return fmt.Errorf("container %d: %w", idx, err) } if err := os.Chmod(conf.ScratchDir, 0777); err != nil { - return err + return fmt.Errorf("container %d: %w", idx, err) } n, err := NewConsulContainer( @@ -138,7 +138,7 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) ports..., ) if err != nil { - return fmt.Errorf("could not add container index %d: %w", idx, err) + return fmt.Errorf("container %d: %w", idx, err) } agents = append(agents, n) c.Index++ @@ -161,9 +161,11 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) func (c *Cluster) Join(agents []Agent) error { return c.join(agents, false) } + func (c *Cluster) JoinExternally(agents []Agent) error { return c.join(agents, true) } + func (c *Cluster) join(agents []Agent, skipSerfJoin bool) error { if len(agents) == 0 { return nil // no change @@ -313,6 +315,16 @@ func (c *Cluster) StandardUpgrade(t *testing.T, ctx context.Context, targetVersi } t.Logf("The number of followers = %d", len(followers)) + // NOTE: we only assert the number of agents in default partition + // TODO: add partition to the cluster struct to assert partition size + clusterSize := 0 + for _, agent := range c.Agents { + if agent.GetPartition() == "" || agent.GetPartition() == "default" { + clusterSize++ + } + } + t.Logf("The number of agents in default partition = %d", clusterSize) + upgradeFn := func(agent Agent, clientFactory func() (*api.Client, error)) error { config := agent.GetConfig() config.Version = targetVersion @@ -347,8 +359,10 @@ func (c *Cluster) StandardUpgrade(t *testing.T, ctx context.Context, targetVersi return err } - // wait until the agent rejoin and leader is elected - WaitForMembers(t, client, len(c.Agents)) + // wait until the agent rejoin and leader is elected; skip non-default agent + if agent.GetPartition() == "" || agent.GetPartition() == "default" { + WaitForMembers(t, client, clusterSize) + } WaitForLeader(t, c, client) return nil @@ -476,7 +490,23 @@ func (c *Cluster) Servers() []Agent { return servers } -// Clients returns the handle to client agents +// Clients returns the handle to client agents in provided partition +func (c *Cluster) ClientsInPartition(partition string) []Agent { + var clients []Agent + + for _, n := range c.Agents { + if n.IsServer() { + continue + } + + if n.GetPartition() == partition { + clients = append(clients, n) + } + } + return clients +} + +// Clients returns the handle to client agents in all partitions func (c *Cluster) Clients() []Agent { var clients []Agent diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 4e577462a185..3ad1b2d35169 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -38,6 +38,7 @@ type consulContainerNode struct { container testcontainers.Container serverMode bool datacenter string + partition string config Config podReq testcontainers.ContainerRequest consulReq testcontainers.ContainerRequest @@ -228,6 +229,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po container: consulContainer, serverMode: pc.Server, datacenter: pc.Datacenter, + partition: pc.Partition, ctx: ctx, podReq: podReq, consulReq: consulReq, @@ -318,6 +320,10 @@ func (c *consulContainerNode) GetDatacenter() string { return c.datacenter } +func (c *consulContainerNode) GetPartition() string { + return c.partition +} + func (c *consulContainerNode) IsServer() bool { return c.serverMode } @@ -493,7 +499,7 @@ func startContainer(ctx context.Context, req testcontainers.ContainerRequest) (t }) } -const pauseImage = "k8s.gcr.io/pause:3.3" +const pauseImage = "registry.k8s.io/pause:3.3" type containerOpts struct { configFile string @@ -641,6 +647,7 @@ type parsedConfig struct { Datacenter string `json:"datacenter"` Server bool `json:"server"` Ports parsedPorts `json:"ports"` + Partition string `json:"partition"` } type parsedPorts struct { diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index b7f617c0df0c..a7385f2e3a0e 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -142,18 +142,24 @@ func (g ConnectContainer) GetStatus() (string, error) { return state.Status, err } +type SidecarConfig struct { + Name string + ServiceID string + Namespace string +} + // NewConnectService returns a container that runs envoy sidecar, launched by // "consul connect envoy", for service name (serviceName) on the specified // node. The container exposes port serviceBindPort and envoy admin port // (19000) by mapping them onto host ports. The container's name has a prefix // combining datacenter and name. -func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID string, serviceBindPorts []int, node cluster.Agent) (*ConnectContainer, error) { +func NewConnectService(ctx context.Context, sidecarCfg SidecarConfig, serviceBindPorts []int, node cluster.Agent) (*ConnectContainer, error) { nodeConfig := node.GetConfig() if nodeConfig.ScratchDir == "" { return nil, fmt.Errorf("node ScratchDir is required") } - namePrefix := fmt.Sprintf("%s-service-connect-%s", node.GetDatacenter(), sidecarServiceName) + namePrefix := fmt.Sprintf("%s-service-connect-%s", node.GetDatacenter(), sidecarCfg.Name) containerName := utils.RandName(namePrefix) envoyVersion := getEnvoyVersion() @@ -181,8 +187,9 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID Name: containerName, Cmd: []string{ "consul", "connect", "envoy", - "-sidecar-for", serviceID, + "-sidecar-for", sidecarCfg.ServiceID, "-admin-bind", fmt.Sprintf("0.0.0.0:%d", internalAdminPort), + "-namespace", sidecarCfg.Namespace, "--", "--log-level", envoyLogLevel, }, @@ -240,7 +247,7 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID ip: info.IP, externalAdminPort: info.MappedPorts[adminPortStr].Int(), internalAdminPort: internalAdminPort, - serviceName: sidecarServiceName, + serviceName: sidecarCfg.Name, } for _, port := range appPortStrs { @@ -248,9 +255,9 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID } fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %v\n", - serviceID, out.appPort, serviceBindPorts) + sidecarCfg.ServiceID, out.appPort, serviceBindPorts) fmt.Printf("NewConnectService sidecar: name %s, mapped admin port %d, admin port %d\n", - sidecarServiceName, out.externalAdminPort, internalAdminPort) + sidecarCfg.Name, out.externalAdminPort, internalAdminPort) return out, nil } diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 8a9b87aa1964..32c899583b9f 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -112,7 +112,6 @@ func (g gatewayContainer) GetPort(port int) (int, error) { return 0, fmt.Errorf("port does not exist") } return p, nil - } func (g gatewayContainer) Restart() error { @@ -140,13 +139,19 @@ func (g gatewayContainer) GetStatus() (string, error) { return state.Status, err } -func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent, ports ...int) (Service, error) { +type GatewayConfig struct { + Name string + Kind string + Namespace string +} + +func NewGatewayService(ctx context.Context, gwCfg GatewayConfig, node libcluster.Agent, ports ...int) (Service, error) { nodeConfig := node.GetConfig() if nodeConfig.ScratchDir == "" { return nil, fmt.Errorf("node ScratchDir is required") } - namePrefix := fmt.Sprintf("%s-service-gateway-%s", node.GetDatacenter(), name) + namePrefix := fmt.Sprintf("%s-service-gateway-%s", node.GetDatacenter(), gwCfg.Name) containerName := utils.RandName(namePrefix) envoyVersion := getEnvoyVersion() @@ -174,9 +179,10 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl Name: containerName, Cmd: []string{ "consul", "connect", "envoy", - fmt.Sprintf("-gateway=%s", kind), + fmt.Sprintf("-gateway=%s", gwCfg.Kind), "-register", - "-service", name, + "-namespace", gwCfg.Namespace, + "-service", gwCfg.Name, "-address", "{{ GetInterfaceIP \"eth0\" }}:8443", "-admin-bind", fmt.Sprintf("0.0.0.0:%d", adminPort), "--", @@ -242,7 +248,7 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl ip: info.IP, port: info.MappedPorts[portStr].Int(), adminPort: info.MappedPorts[adminPortStr].Int(), - serviceName: name, + serviceName: gwCfg.Name, portMappings: portMappings, } diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index 47a8c1364255..3c678f768aef 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -3,6 +3,9 @@ package service import ( "context" "fmt" + "testing" + + "github.com/stretchr/testify/require" "github.com/hashicorp/consul/api" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" @@ -25,13 +28,14 @@ type SidecarService struct { } type ServiceOpts struct { - Name string - ID string - Meta map[string]string - HTTPPort int - GRPCPort int - Checks Checks - Connect SidecarService + Name string + ID string + Meta map[string]string + HTTPPort int + GRPCPort int + Checks Checks + Connect SidecarService + Namespace string } // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers @@ -53,8 +57,12 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int deferClean.Add(func() { _ = serverService.Terminate() }) - - serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", svc.ID), svc.ID, []int{svc.Port}, node) // bindPort not used + sidecarCfg := SidecarConfig{ + Name: fmt.Sprintf("%s-sidecar", svc.ID), + ServiceID: svc.ID, + Namespace: svc.Namespace, + } + serverConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{svc.Port}, node) // bindPort not used if err != nil { return nil, nil, err } @@ -80,6 +88,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts Proxy: &api.AgentServiceConnectProxyConfig{}, }, }, + Namespace: serviceOpts.Namespace, Check: &api.AgentServiceCheck{ Name: "Static Server Listening", TCP: fmt.Sprintf("127.0.0.1:%d", serviceOpts.HTTPPort), @@ -158,7 +167,12 @@ func CreateAndRegisterStaticClientSidecar( } // Create a service and proxy instance - clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, []int{libcluster.ServiceUpstreamLocalBindPort}, node) + sidecarCfg := SidecarConfig{ + Name: fmt.Sprintf("%s-sidecar", StaticClientServiceName), + ServiceID: StaticClientServiceName, + } + + clientConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{libcluster.ServiceUpstreamLocalBindPort}, node) if err != nil { return nil, err } @@ -171,3 +185,59 @@ func CreateAndRegisterStaticClientSidecar( return clientConnectProxy, nil } + +func ClientsCreate(t *testing.T, numClients int, image, version string, cluster *libcluster.Cluster) { + opts := libcluster.BuildOptions{ + ConsulImageName: image, + ConsulVersion: version, + } + ctx := libcluster.NewBuildContext(t, opts) + + conf := libcluster.NewConfigBuilder(ctx). + Client(). + ToAgentConfig(t) + t.Logf("Cluster client config:\n%s", conf.JSON) + + require.NoError(t, cluster.AddN(*conf, numClients, true)) +} + +func ServiceCreate(t *testing.T, client *api.Client, serviceName string) uint64 { + require.NoError(t, client.Agent().ServiceRegister(&api.AgentServiceRegistration{ + Name: serviceName, + Port: 9999, + Connect: &api.AgentServiceConnect{ + SidecarService: &api.AgentServiceRegistration{ + Port: 22005, + }, + }, + })) + + service, meta, err := client.Catalog().Service(serviceName, "", &api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, service, 1) + require.Equal(t, serviceName, service[0].ServiceName) + require.Equal(t, 9999, service[0].ServicePort) + + return meta.LastIndex +} + +func ServiceHealthBlockingQuery(client *api.Client, serviceName string, waitIndex uint64) (chan []*api.ServiceEntry, chan error) { + var ( + ch = make(chan []*api.ServiceEntry, 1) + errCh = make(chan error, 1) + ) + go func() { + opts := &api.QueryOptions{WaitIndex: waitIndex} + service, q, err := client.Health().Service(serviceName, "", false, opts) + if err == nil && q.QueryBackend != api.QueryBackendStreaming { + err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend) + } + if err != nil { + errCh <- err + } else { + ch <- service + } + }() + + return ch, errCh +} diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index ba36978c72f4..0e853f7adda3 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" "github.com/hashicorp/consul/api" @@ -43,22 +44,36 @@ func BasicPeeringTwoClustersSetup( consulVersion string, peeringThroughMeshgateway bool, ) (*BuiltCluster, *BuiltCluster) { - // acceptingCluster, acceptingCtx, acceptingClient := NewPeeringCluster(t, "dc1", 3, consulVersion, true) - acceptingCluster, acceptingCtx, acceptingClient := NewPeeringCluster(t, 3, &libcluster.BuildOptions{ - Datacenter: "dc1", - ConsulVersion: consulVersion, - InjectAutoEncryption: true, + acceptingCluster, acceptingCtx, acceptingClient := NewCluster(t, &ClusterConfig{ + NumServers: 3, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulVersion: consulVersion, + InjectAutoEncryption: true, + }, + ApplyDefaultProxySettings: true, }) - dialingCluster, dialingCtx, dialingClient := NewPeeringCluster(t, 1, &libcluster.BuildOptions{ - Datacenter: "dc2", - ConsulVersion: consulVersion, - InjectAutoEncryption: true, + + dialingCluster, dialingCtx, dialingClient := NewCluster(t, &ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc2", + ConsulVersion: consulVersion, + InjectAutoEncryption: true, + }, + ApplyDefaultProxySettings: true, }) // Create the mesh gateway for dataplane traffic and peering control plane traffic (if enabled) - acceptingClusterGateway, err := libservice.NewGatewayService(context.Background(), "mesh", "mesh", acceptingCluster.Clients()[0]) + gwCfg := libservice.GatewayConfig{ + Name: "mesh", + Kind: "mesh", + } + acceptingClusterGateway, err := libservice.NewGatewayService(context.Background(), gwCfg, acceptingCluster.Clients()[0]) require.NoError(t, err) - dialingClusterGateway, err := libservice.NewGatewayService(context.Background(), "mesh", "mesh", dialingCluster.Clients()[0]) + dialingClusterGateway, err := libservice.NewGatewayService(context.Background(), gwCfg, dialingCluster.Clients()[0]) require.NoError(t, err) // Enable peering control plane traffic through mesh gateway @@ -69,7 +84,7 @@ func BasicPeeringTwoClustersSetup( }, } configCluster := func(cli *api.Client) error { - libassert.CatalogServiceExists(t, cli, "mesh") + libassert.CatalogServiceExists(t, cli, "mesh", nil) ok, _, err := cli.ConfigEntries().Set(req, &api.WriteOptions{}) if !ok { return fmt.Errorf("config entry is not set") @@ -109,8 +124,8 @@ func BasicPeeringTwoClustersSetup( serverService, serverSidecarService, err = libservice.CreateAndRegisterStaticServerAndSidecar(clientNode, &serviceOpts) require.NoError(t, err) - libassert.CatalogServiceExists(t, acceptingClient, libservice.StaticServerServiceName) - libassert.CatalogServiceExists(t, acceptingClient, "static-server-sidecar-proxy") + libassert.CatalogServiceExists(t, acceptingClient, libservice.StaticServerServiceName, nil) + libassert.CatalogServiceExists(t, acceptingClient, "static-server-sidecar-proxy", nil) require.NoError(t, serverService.Export("default", AcceptingPeerName, acceptingClient)) } @@ -125,7 +140,7 @@ func BasicPeeringTwoClustersSetup( clientSidecarService, err = libservice.CreateAndRegisterStaticClientSidecar(clientNode, DialingPeerName, true) require.NoError(t, err) - libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy") + libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy", nil) } @@ -133,7 +148,7 @@ func BasicPeeringTwoClustersSetup( libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", DialingPeerName), "HEALTHY", 1) _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "") return &BuiltCluster{ Cluster: acceptingCluster, @@ -151,95 +166,72 @@ func BasicPeeringTwoClustersSetup( } } -// NewDialingCluster creates a cluster for peering with a single dev agent -// TODO: note: formerly called CreatingPeeringClusterAndSetup -// -// Deprecated: use NewPeeringCluster mostly -func NewDialingCluster( - t *testing.T, - version string, - dialingPeerName string, -) (*libcluster.Cluster, *api.Client, libservice.Service) { - t.Helper() - t.Logf("creating the dialing cluster") - - opts := libcluster.BuildOptions{ - Datacenter: "dc2", - InjectAutoEncryption: true, - InjectGossipEncryption: true, - AllowHTTPAnyway: true, - ConsulVersion: version, - } - ctx := libcluster.NewBuildContext(t, opts) - - conf := libcluster.NewConfigBuilder(ctx). - Peering(true). - ToAgentConfig(t) - t.Logf("dc2 server config: \n%s", conf.JSON) - - cluster, err := libcluster.NewN(t, *conf, 1) - require.NoError(t, err) - - node := cluster.Agents[0] - client := node.GetClient() - libcluster.WaitForLeader(t, cluster, client) - libcluster.WaitForMembers(t, client, 1) - - // Default Proxy Settings - ok, err := utils.ApplyDefaultProxySettings(client) - require.NoError(t, err) - require.True(t, ok) - - // Create the mesh gateway for dataplane traffic - _, err = libservice.NewGatewayService(context.Background(), "mesh", "mesh", node) - require.NoError(t, err) - - // Create a service and proxy instance - clientProxyService, err := libservice.CreateAndRegisterStaticClientSidecar(node, dialingPeerName, true) - require.NoError(t, err) - - libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy") - - return cluster, client, clientProxyService +type ClusterConfig struct { + NumServers int + NumClients int + ApplyDefaultProxySettings bool + BuildOpts *libcluster.BuildOptions + Cmd string + LogConsumer *TestLogConsumer + Ports []int } -// NewPeeringCluster creates a cluster with peering enabled. It also creates +// NewCluster creates a cluster with peering enabled. It also creates // and registers a mesh-gateway at the client agent. The API client returned is // pointed at the client agent. // - proxy-defaults.protocol = tcp -func NewPeeringCluster( +func NewCluster( t *testing.T, - numServers int, - buildOpts *libcluster.BuildOptions, + config *ClusterConfig, ) (*libcluster.Cluster, *libcluster.BuildContext, *api.Client) { - require.NotEmpty(t, buildOpts.Datacenter) - require.True(t, numServers > 0) + var ( + cluster *libcluster.Cluster + err error + ) + require.NotEmpty(t, config.BuildOpts.Datacenter) + require.True(t, config.NumServers > 0) opts := libcluster.BuildOptions{ - Datacenter: buildOpts.Datacenter, - InjectAutoEncryption: buildOpts.InjectAutoEncryption, + Datacenter: config.BuildOpts.Datacenter, + InjectAutoEncryption: config.BuildOpts.InjectAutoEncryption, InjectGossipEncryption: true, AllowHTTPAnyway: true, - ConsulVersion: buildOpts.ConsulVersion, - ACLEnabled: buildOpts.ACLEnabled, + ConsulVersion: config.BuildOpts.ConsulVersion, + ACLEnabled: config.BuildOpts.ACLEnabled, } ctx := libcluster.NewBuildContext(t, opts) serverConf := libcluster.NewConfigBuilder(ctx). - Bootstrap(numServers). + Bootstrap(config.NumServers). Peering(true). ToAgentConfig(t) t.Logf("%s server config: \n%s", opts.Datacenter, serverConf.JSON) - cluster, err := libcluster.NewN(t, *serverConf, numServers) + // optional + if config.LogConsumer != nil { + serverConf.LogConsumer = config.LogConsumer + } + + t.Logf("Cluster config:\n%s", serverConf.JSON) + + // optional custom cmd + if config.Cmd != "" { + serverConf.Cmd = append(serverConf.Cmd, config.Cmd) + } + + if config.Ports != nil { + cluster, err = libcluster.New(t, []libcluster.Config{*serverConf}, config.Ports...) + } else { + cluster, err = libcluster.NewN(t, *serverConf, config.NumServers) + } require.NoError(t, err) var retryJoin []string - for i := 0; i < numServers; i++ { + for i := 0; i < config.NumServers; i++ { retryJoin = append(retryJoin, fmt.Sprintf("agent-%d", i)) } - // Add a stable client to register the service + // Add numClients static clients to register the service configbuiilder := libcluster.NewConfigBuilder(ctx). Client(). Peering(true). @@ -247,18 +239,33 @@ func NewPeeringCluster( clientConf := configbuiilder.ToAgentConfig(t) t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON) - require.NoError(t, cluster.AddN(*clientConf, 1, true)) + require.NoError(t, cluster.AddN(*clientConf, config.NumClients, true)) // Use the client agent as the HTTP endpoint since we will not rotate it in many tests. - clientNode := cluster.Agents[numServers] - client := clientNode.GetClient() + var client *api.Client + if config.NumClients > 0 { + clientNode := cluster.Agents[config.NumServers] + client = clientNode.GetClient() + } else { + client = cluster.Agents[0].GetClient() + } libcluster.WaitForLeader(t, cluster, client) - libcluster.WaitForMembers(t, client, numServers+1) + libcluster.WaitForMembers(t, client, config.NumServers+config.NumClients) // Default Proxy Settings - ok, err := utils.ApplyDefaultProxySettings(client) - require.NoError(t, err) - require.True(t, ok) + if config.ApplyDefaultProxySettings { + ok, err := utils.ApplyDefaultProxySettings(client) + require.NoError(t, err) + require.True(t, ok) + } return cluster, ctx, client } + +type TestLogConsumer struct { + Msgs []string +} + +func (g *TestLogConsumer) Accept(l testcontainers.Log) { + g.Msgs = append(g.Msgs, string(l.Content)) +} diff --git a/test/integration/consul-container/libs/topology/service_topology.go b/test/integration/consul-container/libs/topology/service_topology.go index 1cacd1a84c40..25ef6179ec7b 100644 --- a/test/integration/consul-container/libs/topology/service_topology.go +++ b/test/integration/consul-container/libs/topology/service_topology.go @@ -38,14 +38,14 @@ func CreateServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName)) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName), nil) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) // Create a client proxy instance with the server as an upstream clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName), nil) return serverConnectProxy, clientConnectProxy } diff --git a/test/integration/consul-container/test/basic/connect_service_test.go b/test/integration/consul-container/test/basic/connect_service_test.go index 90a80c84c6db..655f90a1d0c1 100644 --- a/test/integration/consul-container/test/basic/connect_service_test.go +++ b/test/integration/consul-container/test/basic/connect_service_test.go @@ -34,7 +34,7 @@ func TestBasicConnectService(t *testing.T) { libassert.AssertContainerState(t, clientService, "running") libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") } func createCluster(t *testing.T) *libcluster.Cluster { @@ -84,14 +84,14 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) libservice.Servic _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy") - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) // Create a client proxy instance with the server as an upstream clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy") + libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil) return clientConnectProxy } diff --git a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go index 2aa81954f8d8..da642f183416 100644 --- a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go +++ b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go @@ -3,82 +3,121 @@ package gateways import ( "context" "fmt" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/sdk/testutil/retry" - libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" - libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" - "github.com/hashicorp/go-cleanhttp" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "io" "net/http" "strings" "testing" "time" -) -var ( - checkTimeout = 1 * time.Minute - checkInterval = 1 * time.Second + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/go-cleanhttp" ) // Creates a gateway service and tests to see if it is routable func TestAPIGatewayCreate(t *testing.T) { - t.Skip() if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() + gatewayName := randomName("gateway", 16) + routeName := randomName("route", 16) + serviceName := randomName("service", 16) listenerPortOne := 6000 + serviceHTTPPort := 6001 + serviceGRPCPort := 6002 + + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerPortOne, + serviceHTTPPort, + serviceGRPCPort, + }, + } - cluster := createCluster(t, listenerPortOne) - + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) client := cluster.APIClient(0) - //setup + namespace := getNamespace() + if namespace != "" { + ns := &api.Namespace{Name: namespace} + _, _, err := client.Namespaces().Create(ns, nil) + require.NoError(t, err) + } + + // add api gateway config apiGateway := &api.APIGatewayConfigEntry{ - Kind: "api-gateway", - Name: "api-gateway", + Kind: api.APIGateway, + Namespace: namespace, + Name: gatewayName, Listeners: []api.APIGatewayListener{ { + Name: "listener", Port: listenerPortOne, Protocol: "tcp", }, }, } - _, _, err := client.ConfigEntries().Set(apiGateway, nil) + + require.NoError(t, cluster.ConfigEntryWrite(apiGateway)) + + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceName, + Name: serviceName, + Namespace: namespace, + HTTPPort: serviceHTTPPort, + GRPCPort: serviceGRPCPort, + }) require.NoError(t, err) tcpRoute := &api.TCPRouteConfigEntry{ - Kind: "tcp-route", - Name: "api-gateway-route", + Kind: api.TCPRoute, + Name: routeName, + Namespace: namespace, Parents: []api.ResourceReference{ { - Kind: "api-gateway", - Name: "api-gateway", + Kind: api.APIGateway, + Namespace: namespace, + Name: gatewayName, }, }, Services: []api.TCPService{ { - Name: libservice.StaticServerServiceName, + Namespace: namespace, + Name: serviceName, }, }, } - _, _, err = client.ConfigEntries().Set(tcpRoute, nil) - require.NoError(t, err) + require.NoError(t, cluster.ConfigEntryWrite(tcpRoute)) - // Create a client proxy instance with the server as an upstream - _, gatewayService := createServices(t, cluster, listenerPortOne) + // Create a gateway + gatewayService, err := libservice.NewGatewayService(context.Background(), libservice.GatewayConfig{ + Kind: "api", + Namespace: namespace, + Name: gatewayName, + }, cluster.Agents[0], listenerPortOne) + require.NoError(t, err) - //make sure the gateway/route come online - //make sure config entries have been properly created - checkGatewayConfigEntry(t, client, "api-gateway", "") - checkTCPRouteConfigEntry(t, client, "api-gateway-route", "") + // make sure the gateway/route come online + // make sure config entries have been properly created + checkGatewayConfigEntry(t, client, gatewayName, namespace) + checkTCPRouteConfigEntry(t, client, routeName, namespace) port, err := gatewayService.GetPort(listenerPortOne) require.NoError(t, err) @@ -102,71 +141,36 @@ func conditionStatusIsValue(typeName string, statusValue string, conditions []ap return false } -// TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some -func createCluster(t *testing.T, ports ...int) *libcluster.Cluster { - opts := libcluster.BuildOptions{ - InjectAutoEncryption: true, - InjectGossipEncryption: true, - AllowHTTPAnyway: true, - } - ctx := libcluster.NewBuildContext(t, opts) - - conf := libcluster.NewConfigBuilder(ctx). - ToAgentConfig(t) - t.Logf("Cluster config:\n%s", conf.JSON) - - configs := []libcluster.Config{*conf} - - cluster, err := libcluster.New(t, configs, ports...) - require.NoError(t, err) - - node := cluster.Agents[0] - client := node.GetClient() - - libcluster.WaitForLeader(t, cluster, client) - libcluster.WaitForMembers(t, client, 1) - - // Default Proxy Settings - ok, err := utils.ApplyDefaultProxySettings(client) - require.NoError(t, err) - require.True(t, ok) - - require.NoError(t, err) - - return cluster -} - -func createGateway(gatewayName string, protocol string, listenerPort int) *api.APIGatewayConfigEntry { - return &api.APIGatewayConfigEntry{ - Kind: api.APIGateway, - Name: gatewayName, - Listeners: []api.APIGatewayListener{ - { - Name: "listener", - Port: listenerPort, - Protocol: protocol, - }, - }, - } -} - func checkGatewayConfigEntry(t *testing.T, client *api.Client, gatewayName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } if entry == nil { + t.Log("returned entry is nil") return false } + apiEntry := entry.(*api.APIGatewayConfigEntry) return isAccepted(apiEntry.Status.Conditions) }, time.Second*10, time.Second*1) } func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } if entry == nil { + t.Log("returned entry is nil") return false } @@ -176,10 +180,16 @@ func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName strin } func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.TCPRoute, routeName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } if entry == nil { + t.Log("returned entry is nil") return false } @@ -188,39 +198,6 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string }, time.Second*10, time.Second*1) } -func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service { - node := cluster.Agents[0] - client := node.GetClient() - // Create a service and proxy instance - service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...) - require.NoError(t, err) - - libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy") - libassert.CatalogServiceExists(t, client, serviceOpts.Name) - - return service - -} -func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) { - node := cluster.Agents[0] - client := node.GetClient() - // Create a service and proxy instance - serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", - HTTPPort: 8080, - GRPCPort: 8079, - } - - clientConnectProxy := createService(t, cluster, serviceOpts, nil) - - gatewayService, err := libservice.NewGatewayService(context.Background(), "api-gateway", "api", cluster.Agents[0], ports...) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "api-gateway") - - return clientConnectProxy, gatewayService -} - type checkOptions struct { debug bool statusCode int @@ -229,26 +206,23 @@ type checkOptions struct { // checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances func checkRoute(t *testing.T, port int, path string, headers map[string]string, expected checkOptions) { - ip := "localhost" + t.Helper() + if expected.testName != "" { t.Log("running " + expected.testName) } - const phrase = "hello" - - failer := func() *retry.Timer { - return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} - } client := cleanhttp.DefaultClient() - path = strings.TrimPrefix(path, "/") - url := fmt.Sprintf("http://%s:%d/%s", ip, port, path) + url := fmt.Sprintf("http://localhost:%d/%s", port, path) - retry.RunWith(failer(), t, func(r *retry.R) { - t.Logf("making call to %s", url) - reader := strings.NewReader(phrase) + require.Eventually(t, func() bool { + reader := strings.NewReader("hello") req, err := http.NewRequest("POST", url, reader) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } headers["content-type"] = "text/plain" for k, v := range headers { @@ -258,40 +232,41 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string, req.Host = v } } + res, err := client.Do(req) if err != nil { - t.Log(err) - r.Fatal("could not make call to service ", url) + t.Log("error sending request", err) + return false } defer res.Body.Close() body, err := io.ReadAll(res.Body) if err != nil { - r.Fatal("could not read response body ", url) + t.Log("error reading response body", err) + return false } - assert.Equal(t, expected.statusCode, res.StatusCode) if expected.statusCode != res.StatusCode { - r.Fatal("unexpected response code returned") + t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode) + return false } - - //if debug is expected, debug should be in the response body - assert.Equal(t, expected.debug, strings.Contains(string(body), "debug")) - if expected.statusCode != res.StatusCode { - r.Fatal("unexpected response body returned") + if expected.debug { + if !strings.Contains(string(body), "debug") { + t.Log("body does not contain 'debug'") + return false + } } - - if !strings.Contains(string(body), phrase) { - r.Fatal("received an incorrect response ", string(body)) + if !strings.Contains(string(body), "hello") { + t.Log("body does not contain 'hello'") + return false } - }) + return true + }, time.Second*30, time.Second*1) } func checkRouteError(t *testing.T, ip string, port int, path string, headers map[string]string, expected string) { - failer := func() *retry.Timer { - return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} - } + t.Helper() client := cleanhttp.DefaultClient() url := fmt.Sprintf("http://%s:%d", ip, port) @@ -300,11 +275,12 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map url += "/" + path } - retry.RunWith(failer(), t, func(r *retry.R) { - t.Logf("making call to %s", url) + require.Eventually(t, func() bool { req, err := http.NewRequest("GET", url, nil) - assert.NoError(t, err) - + if err != nil { + t.Log("error constructing request", err) + return false + } for k, v := range headers { req.Header.Set(k, v) @@ -313,10 +289,16 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map } } _, err = client.Do(req) - assert.Error(t, err) - + if err == nil { + t.Log("client request should have errored, but didn't") + return false + } if expected != "" { - assert.ErrorContains(t, err, expected) + if !strings.Contains(err.Error(), expected) { + t.Logf("expected %q to contain %q", err.Error(), expected) + return false + } } - }) + return true + }, time.Second*30, time.Second*1) } diff --git a/test/integration/consul-container/test/gateways/http_route_test.go b/test/integration/consul-container/test/gateways/http_route_test.go index 61343f34950d..a1a6c5a0e77f 100644 --- a/test/integration/consul-container/test/gateways/http_route_test.go +++ b/test/integration/consul-container/test/gateways/http_route_test.go @@ -5,19 +5,19 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/api" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" - "time" + libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" ) -func getNamespace() string { - return "" -} - // randomName generates a random name of n length with the provided // prefix. If prefix is omitted, the then entire name is random char. func randomName(prefix string, n int) string { @@ -36,54 +36,89 @@ func TestHTTPRouteFlattening(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } - t.Parallel() - //infrastructure set up - listenerPort := 6000 - //create cluster - cluster := createCluster(t, listenerPort) - client := cluster.Agents[0].GetClient() - service1ResponseCode := 200 - service2ResponseCode := 418 - serviceOne := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service1", - ID: "service1", - HTTPPort: 8080, - GRPCPort: 8079, - }, []string{ - //customizes response code so we can distinguish between which service is responding - "-echo-server-default-params", fmt.Sprintf("status=%d", service1ResponseCode), - }) - serviceTwo := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service2", - ID: "service2", - HTTPPort: 8081, - GRPCPort: 8082, - }, []string{ - "-echo-server-default-params", fmt.Sprintf("status=%d", service2ResponseCode), - }, - ) + t.Parallel() - //TODO this should only matter in consul enterprise I believe? - namespace := getNamespace() + // infrastructure set up + listenerPort := 6004 + serviceOneHTTPPort := 6005 + serviceOneGRPCPort := 6006 + serviceTwoHTTPPort := 6007 + serviceTwoGRPCPort := 6008 + + serviceOneName := randomName("service", 16) + serviceTwoName := randomName("service", 16) + serviceOneResponseCode := 200 + serviceTwoResponseCode := 418 gatewayName := randomName("gw", 16) routeOneName := randomName("route", 16) routeTwoName := randomName("route", 16) path1 := "/" path2 := "/v2" - //write config entries - proxyDefaults := &api.ProxyConfigEntry{ - Kind: api.ProxyDefaults, - Name: api.ProxyConfigGlobal, + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerPort, + serviceOneHTTPPort, + serviceOneGRPCPort, + serviceTwoHTTPPort, + serviceTwoGRPCPort, + }, + ApplyDefaultProxySettings: true, + } + + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) + client := cluster.Agents[0].GetClient() + + namespace := getNamespace() + if namespace != "" { + ns := &api.Namespace{Name: namespace} + _, _, err := client.Namespaces().Create(ns, nil) + require.NoError(t, err) + } + + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceOneName, + Name: serviceOneName, + Namespace: namespace, + HTTPPort: serviceOneHTTPPort, + GRPCPort: serviceOneGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding + "-echo-server-default-params", fmt.Sprintf("status=%d", serviceOneResponseCode), + ) + require.NoError(t, err) + + _, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceTwoName, + Name: serviceTwoName, Namespace: namespace, + HTTPPort: serviceTwoHTTPPort, + GRPCPort: serviceTwoGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding + "-echo-server-default-params", fmt.Sprintf("status=%d", serviceTwoResponseCode), + ) + require.NoError(t, err) + + // write config entries + proxyDefaults := &api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, Config: map[string]interface{}{ "protocol": "http", }, } - _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) - require.NoError(t, err) + require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults)) apiGateway := &api.APIGatewayConfigEntry{ Kind: "api-gateway", @@ -95,11 +130,13 @@ func TestHTTPRouteFlattening(t *testing.T) { Protocol: "http", }, }, + Namespace: namespace, } routeOne := &api.HTTPRouteConfigEntry{ - Kind: api.HTTPRoute, - Name: routeOneName, + Kind: api.HTTPRoute, + Name: routeOneName, + Namespace: namespace, Parents: []api.ResourceReference{ { Kind: api.APIGateway, @@ -111,12 +148,11 @@ func TestHTTPRouteFlattening(t *testing.T) { "test.foo", "test.example", }, - Namespace: namespace, Rules: []api.HTTPRouteRule{ { Services: []api.HTTPService{ { - Name: serviceOne.GetServiceName(), + Name: serviceOneName, Namespace: namespace, }, }, @@ -133,8 +169,9 @@ func TestHTTPRouteFlattening(t *testing.T) { } routeTwo := &api.HTTPRouteConfigEntry{ - Kind: api.HTTPRoute, - Name: routeTwoName, + Kind: api.HTTPRoute, + Name: routeTwoName, + Namespace: namespace, Parents: []api.ResourceReference{ { Kind: api.APIGateway, @@ -145,12 +182,11 @@ func TestHTTPRouteFlattening(t *testing.T) { Hostnames: []string{ "test.foo", }, - Namespace: namespace, Rules: []api.HTTPRouteRule{ { Services: []api.HTTPService{ { - Name: serviceTwo.GetServiceName(), + Name: serviceTwoName, Namespace: namespace, }, }, @@ -173,59 +209,60 @@ func TestHTTPRouteFlattening(t *testing.T) { }, } - _, _, err = client.ConfigEntries().Set(apiGateway, nil) - require.NoError(t, err) - _, _, err = client.ConfigEntries().Set(routeOne, nil) - require.NoError(t, err) - _, _, err = client.ConfigEntries().Set(routeTwo, nil) - require.NoError(t, err) + require.NoError(t, cluster.ConfigEntryWrite(apiGateway)) + require.NoError(t, cluster.ConfigEntryWrite(routeOne)) + require.NoError(t, cluster.ConfigEntryWrite(routeTwo)) - //create gateway service - gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort) + // create gateway service + gwCfg := libservice.GatewayConfig{ + Name: gatewayName, + Kind: "api", + Namespace: namespace, + } + gatewayService, err := libservice.NewGatewayService(context.Background(), gwCfg, cluster.Agents[0], listenerPort) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, gatewayName) + libassert.CatalogServiceExists(t, client, gatewayName, &api.QueryOptions{Namespace: namespace}) - //make sure config entries have been properly created + // make sure config entries have been properly created checkGatewayConfigEntry(t, client, gatewayName, namespace) + t.Log("checking route one") checkHTTPRouteConfigEntry(t, client, routeOneName, namespace) checkHTTPRouteConfigEntry(t, client, routeTwoName, namespace) - //gateway resolves routes + // gateway resolves routes gatewayPort, err := gatewayService.GetPort(listenerPort) require.NoError(t, err) + fmt.Println("Gateway Port: ", gatewayPort) - //route 2 with headers - - //Same v2 path with and without header + // Same v2 path with and without header checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 header and path"}) checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.foo", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just path match"}) - ////v1 path with the header + // //v1 path with the header checkRoute(t, gatewayPort, "/check", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just header match"}) checkRoute(t, gatewayPort, "/v2/path/value", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 v2 with path"}) - //hit service 1 by hitting root path + // hit service 1 by hitting root path checkRoute(t, gatewayPort, "", map[string]string{ "Host": "test.foo", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"}) + }, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1 root prefix"}) - //hit service 1 by hitting v2 path with v1 hostname + // hit service 1 by hitting v2 path with v1 hostname checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.example", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"}) - + }, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"}) } func TestHTTPRoutePathRewrite(t *testing.T) { @@ -235,59 +272,107 @@ func TestHTTPRoutePathRewrite(t *testing.T) { t.Parallel() - //infrastructure set up - listenerPort := 6001 - //create cluster - cluster := createCluster(t, listenerPort) - client := cluster.Agents[0].GetClient() + // infrastructure set up + listenerPort := 6009 + fooHTTPPort := 6010 + fooGRPCPort := 6011 + barHTTPPort := 6012 + barGRPCPort := 6013 + + fooName := randomName("foo", 16) + barName := randomName("bar", 16) + gatewayName := randomName("gw", 16) + invalidRouteName := randomName("route", 16) + validRouteName := randomName("route", 16) + + // create cluster + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerPort, + fooHTTPPort, + fooGRPCPort, + barHTTPPort, + barGRPCPort, + }, + ApplyDefaultProxySettings: true, + } + + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) + client := cluster.APIClient(0) + fooStatusCode := 400 barStatusCode := 201 fooPath := "/v1/foo" barPath := "/v1/bar" - fooService := createService(t, cluster, &libservice.ServiceOpts{ - Name: "foo", - ID: "foo", - HTTPPort: 8080, - GRPCPort: 8081, - }, []string{ - //customizes response code so we can distinguish between which service is responding + namespace := getNamespace() + if namespace != "" { + ns := &api.Namespace{Name: namespace} + _, _, err := client.Namespaces().Create(ns, nil) + require.NoError(t, err) + } + + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: fooName, + Name: fooName, + Namespace: namespace, + HTTPPort: fooHTTPPort, + GRPCPort: fooGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding "-echo-debug-path", fooPath, "-echo-server-default-params", fmt.Sprintf("status=%d", fooStatusCode), - }) - barService := createService(t, cluster, &libservice.ServiceOpts{ - Name: "bar", - ID: "bar", - //TODO we can potentially get conflicts if these ports are the same - HTTPPort: 8079, - GRPCPort: 8078, - }, []string{ + ) + require.NoError(t, err) + + _, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: barName, + Name: barName, + Namespace: namespace, + HTTPPort: barHTTPPort, + GRPCPort: barGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding "-echo-debug-path", barPath, "-echo-server-default-params", fmt.Sprintf("status=%d", barStatusCode), - }, ) + require.NoError(t, err) - namespace := getNamespace() - gatewayName := randomName("gw", 16) - invalidRouteName := randomName("route", 16) - validRouteName := randomName("route", 16) fooUnrewritten := "/foo" barUnrewritten := "/bar" - //write config entries + // write config entries proxyDefaults := &api.ProxyConfigEntry{ Kind: api.ProxyDefaults, Name: api.ProxyConfigGlobal, - Namespace: namespace, + Namespace: "", // proxy-defaults can only be set in the default namespace Config: map[string]interface{}{ "protocol": "http", }, } - _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) - require.NoError(t, err) + require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults)) - apiGateway := createGateway(gatewayName, "http", listenerPort) + apiGateway := &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: gatewayName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerPort, + Protocol: "http", + }, + }, + Namespace: namespace, + } fooRoute := &api.HTTPRouteConfigEntry{ Kind: api.HTTPRoute, @@ -312,7 +397,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) { }, Services: []api.HTTPService{ { - Name: fooService.GetServiceName(), + Name: fooName, Namespace: namespace, }, }, @@ -351,7 +436,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) { }, Services: []api.HTTPService{ { - Name: barService.GetServiceName(), + Name: barName, Namespace: namespace, }, }, @@ -367,19 +452,21 @@ func TestHTTPRoutePathRewrite(t *testing.T) { }, } - _, _, err = client.ConfigEntries().Set(apiGateway, nil) - require.NoError(t, err) - _, _, err = client.ConfigEntries().Set(fooRoute, nil) - require.NoError(t, err) - _, _, err = client.ConfigEntries().Set(barRoute, nil) - require.NoError(t, err) + require.NoError(t, cluster.ConfigEntryWrite(apiGateway)) + require.NoError(t, cluster.ConfigEntryWrite(fooRoute)) + require.NoError(t, cluster.ConfigEntryWrite(barRoute)) - //create gateway service - gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort) + // create gateway service + gwCfg := libservice.GatewayConfig{ + Name: gatewayName, + Kind: "api", + Namespace: namespace, + } + gatewayService, err := libservice.NewGatewayService(context.Background(), gwCfg, cluster.Agents[0], listenerPort) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, gatewayName) + libassert.CatalogServiceExists(t, client, gatewayName, &api.QueryOptions{Namespace: namespace}) - //make sure config entries have been properly created + // make sure config entries have been properly created checkGatewayConfigEntry(t, client, gatewayName, namespace) checkHTTPRouteConfigEntry(t, client, invalidRouteName, namespace) checkHTTPRouteConfigEntry(t, client, validRouteName, namespace) @@ -387,71 +474,100 @@ func TestHTTPRoutePathRewrite(t *testing.T) { gatewayPort, err := gatewayService.GetPort(listenerPort) require.NoError(t, err) - //TODO these were the assertions we had in the original test. potentially would want more test cases + // TODO these were the assertions we had in the original test. potentially would want more test cases - //NOTE: Hitting the debug path code overrides default expected value + // NOTE: Hitting the debug path code overrides default expected value debugExpectedStatusCode := 200 - //hit foo, making sure path is being rewritten by hitting the debug page + // hit foo, making sure path is being rewritten by hitting the debug page checkRoute(t, gatewayPort, fooUnrewritten, map[string]string{ "Host": "test.foo", }, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "foo service"}) - //make sure foo is being sent to proper service + // make sure foo is being sent to proper service checkRoute(t, gatewayPort, fooUnrewritten+"/foo", map[string]string{ "Host": "test.foo", - }, checkOptions{debug: false, statusCode: fooStatusCode, testName: "foo service"}) + }, checkOptions{debug: false, statusCode: fooStatusCode, testName: "foo service 2"}) - //hit bar, making sure its been rewritten + // hit bar, making sure its been rewritten checkRoute(t, gatewayPort, barUnrewritten, map[string]string{ "Host": "test.foo", }, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "bar service"}) - //hit bar, making sure its being sent to the proper service + // hit bar, making sure its being sent to the proper service checkRoute(t, gatewayPort, barUnrewritten+"/bar", map[string]string{ "Host": "test.foo", }, checkOptions{debug: false, statusCode: barStatusCode, testName: "bar service"}) - } func TestHTTPRouteParentRefChange(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } + t.Parallel() // infrastructure set up address := "localhost" - listenerOnePort := 6000 - listenerTwoPort := 6001 + listenerOnePort := 6014 + listenerTwoPort := 6015 + serviceHTTPPort := 6016 + serviceGRPCPort := 6017 - // create cluster and service - cluster := createCluster(t, listenerOnePort, listenerTwoPort) - client := cluster.Agents[0].GetClient() - service := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service", - ID: "service", - HTTPPort: 8080, - GRPCPort: 8079, - }, []string{}) - - // getNamespace() should always return an empty string in Consul OSS - namespace := getNamespace() + serviceName := randomName("service", 16) gatewayOneName := randomName("gw1", 16) gatewayTwoName := randomName("gw2", 16) routeName := randomName("route", 16) + // create cluster + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerOnePort, + listenerTwoPort, + serviceHTTPPort, + serviceGRPCPort, + }, + ApplyDefaultProxySettings: true, + } + + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) + client := cluster.APIClient(0) + + // getNamespace() should always return an empty string in Consul OSS + namespace := getNamespace() + if namespace != "" { + ns := &api.Namespace{Name: namespace} + _, _, err := client.Namespaces().Create(ns, nil) + require.NoError(t, err) + } + + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceName, + Name: serviceName, + Namespace: namespace, + HTTPPort: serviceHTTPPort, + GRPCPort: serviceGRPCPort, + }) + require.NoError(t, err) + // write config entries proxyDefaults := &api.ProxyConfigEntry{ - Kind: api.ProxyDefaults, - Name: api.ProxyConfigGlobal, - Namespace: namespace, + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, Config: map[string]interface{}{ "protocol": "http", }, } - _, _, err := client.ConfigEntries().Set(proxyDefaults, nil) - assert.NoError(t, err) + + require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults)) // create gateway config entry gatewayOne := &api.APIGatewayConfigEntry{ @@ -465,24 +581,20 @@ func TestHTTPRouteParentRefChange(t *testing.T) { Hostname: "test.foo", }, }, + Namespace: namespace, } - _, _, err = client.ConfigEntries().Set(gatewayOne, nil) - assert.NoError(t, err) - require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace}) - assert.NoError(t, err) - if entry == nil { - return false - } - apiEntry := entry.(*api.APIGatewayConfigEntry) - t.Log(entry) - return isAccepted(apiEntry.Status.Conditions) - }, time.Second*10, time.Second*1) + require.NoError(t, cluster.ConfigEntryWrite(gatewayOne)) + checkGatewayConfigEntry(t, client, gatewayOneName, namespace) // create gateway service - gatewayOneService, err := libservice.NewGatewayService(context.Background(), gatewayOneName, "api", cluster.Agents[0], listenerOnePort) + gwOneCfg := libservice.GatewayConfig{ + Name: gatewayOneName, + Kind: "api", + Namespace: namespace, + } + gatewayOneService, err := libservice.NewGatewayService(context.Background(), gwOneCfg, cluster.Agents[0], listenerOnePort) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, gatewayOneName) + libassert.CatalogServiceExists(t, client, gatewayOneName, &api.QueryOptions{Namespace: namespace}) // create gateway config entry gatewayTwo := &api.APIGatewayConfigEntry{ @@ -496,24 +608,20 @@ func TestHTTPRouteParentRefChange(t *testing.T) { Hostname: "test.example", }, }, + Namespace: namespace, } - _, _, err = client.ConfigEntries().Set(gatewayTwo, nil) - assert.NoError(t, err) - require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace}) - assert.NoError(t, err) - if entry == nil { - return false - } - apiEntry := entry.(*api.APIGatewayConfigEntry) - t.Log(entry) - return isAccepted(apiEntry.Status.Conditions) - }, time.Second*10, time.Second*1) + require.NoError(t, cluster.ConfigEntryWrite(gatewayTwo)) + checkGatewayConfigEntry(t, client, gatewayTwoName, namespace) // create gateway service - gatewayTwoService, err := libservice.NewGatewayService(context.Background(), gatewayTwoName, "api", cluster.Agents[0], listenerTwoPort) + gwTwoCfg := libservice.GatewayConfig{ + Name: gatewayTwoName, + Kind: "api", + Namespace: namespace, + } + gatewayTwoService, err := libservice.NewGatewayService(context.Background(), gwTwoCfg, cluster.Agents[0], listenerTwoPort) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, gatewayTwoName) + libassert.CatalogServiceExists(t, client, gatewayTwoName, &api.QueryOptions{Namespace: namespace}) // create route to service, targeting first gateway route := &api.HTTPRouteConfigEntry{ @@ -535,7 +643,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) { { Services: []api.HTTPService{ { - Name: service.GetServiceName(), + Name: serviceName, Namespace: namespace, }, }, @@ -550,8 +658,9 @@ func TestHTTPRouteParentRefChange(t *testing.T) { }, }, } - _, _, err = client.ConfigEntries().Set(route, nil) - assert.NoError(t, err) + + require.NoError(t, cluster.ConfigEntryWrite(route)) + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) assert.NoError(t, err) @@ -593,8 +702,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) { Namespace: namespace, }, } - _, _, err = client.ConfigEntries().Set(route, nil) - assert.NoError(t, err) + + require.NoError(t, cluster.ConfigEntryWrite(route)) require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) assert.NoError(t, err) diff --git a/test/integration/consul-container/test/gateways/namespace_oss.go b/test/integration/consul-container/test/gateways/namespace_oss.go new file mode 100644 index 000000000000..3867e72987bc --- /dev/null +++ b/test/integration/consul-container/test/gateways/namespace_oss.go @@ -0,0 +1,8 @@ +//go:build !consulent +// +build !consulent + +package gateways + +func getNamespace() string { + return "" +} diff --git a/test/integration/consul-container/test/observability/access_logs_test.go b/test/integration/consul-container/test/observability/access_logs_test.go index 6c173e7c0351..37a80c3563e0 100644 --- a/test/integration/consul-container/test/observability/access_logs_test.go +++ b/test/integration/consul-container/test/observability/access_logs_test.go @@ -45,9 +45,14 @@ func TestAccessLogs(t *testing.T) { t.Skip() } - cluster, _, _ := topology.NewPeeringCluster(t, 1, &libcluster.BuildOptions{ - Datacenter: "dc1", - InjectAutoEncryption: true, + cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + ApplyDefaultProxySettings: true, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + }, }) // Turn on access logs. Do this before starting the sidecars so that they inherit the configuration @@ -70,7 +75,7 @@ func TestAccessLogs(t *testing.T) { // Validate Custom JSON require.Eventually(t, func() bool { libassert.HTTPServiceEchoes(t, "localhost", port, "banana") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") client := libassert.ServiceLogContains(t, clientService, "\"banana_path\":\"/banana\"") server := libassert.ServiceLogContains(t, serverService, "\"banana_path\":\"/banana\"") return client && server @@ -112,7 +117,7 @@ func TestAccessLogs(t *testing.T) { _, port = clientService.GetAddr() require.Eventually(t, func() bool { libassert.HTTPServiceEchoes(t, "localhost", port, "orange") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") client := libassert.ServiceLogContains(t, clientService, "Orange you glad I didn't say banana: /orange, -") server := libassert.ServiceLogContains(t, serverService, "Orange you glad I didn't say banana: /orange, -") return client && server diff --git a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go index bbac9cc03401..5081433e2495 100644 --- a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go +++ b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go @@ -94,7 +94,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") } testutil.RunStep(t, "rotate exporting cluster's root CA", func(t *testing.T) { @@ -144,7 +144,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { // Connectivity should still be contained _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") verifySidecarHasTwoRootCAs(t, clientSidecarService) }) @@ -166,7 +166,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") }) } diff --git a/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go index 3470e738922d..92f5583381ff 100644 --- a/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go +++ b/test/integration/consul-container/test/troubleshoot/troubleshoot_upstream_test.go @@ -18,9 +18,14 @@ import ( func TestTroubleshootProxy(t *testing.T) { t.Parallel() - cluster, _, _ := topology.NewPeeringCluster(t, 1, &libcluster.BuildOptions{ - Datacenter: "dc1", - InjectAutoEncryption: true, + cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + }, + ApplyDefaultProxySettings: true, }) serverService, clientService := topology.CreateServices(t, cluster) diff --git a/test/integration/consul-container/test/upgrade/acl_node_test.go b/test/integration/consul-container/test/upgrade/acl_node_test.go index 26095cf17480..2ad304c52789 100644 --- a/test/integration/consul-container/test/upgrade/acl_node_test.go +++ b/test/integration/consul-container/test/upgrade/acl_node_test.go @@ -36,11 +36,16 @@ func TestACL_Upgrade_Node_Token(t *testing.T) { run := func(t *testing.T, tc testcase) { // NOTE: Disable auto.encrypt due to its conflict with ACL token during bootstrap - cluster, _, _ := libtopology.NewPeeringCluster(t, 1, &libcluster.BuildOptions{ - Datacenter: "dc1", - ConsulVersion: tc.oldversion, - InjectAutoEncryption: false, - ACLEnabled: true, + cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulVersion: tc.oldversion, + InjectAutoEncryption: false, + ACLEnabled: true, + }, + ApplyDefaultProxySettings: true, }) agentToken, err := cluster.CreateAgentToken("dc1", diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go index 2203954a44d3..bcbefe937ee8 100644 --- a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go @@ -67,7 +67,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { } _, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-server") + libassert.CatalogServiceExists(t, client, "static-server", nil) // TODO: verify the number of instance of static-server is 3 libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 3) @@ -121,7 +121,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2", "") }, }, { @@ -194,7 +194,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { libassert.AssertUpstreamEndpointStatus(t, adminPort, "test.static-server.default", "UNHEALTHY", 1) // static-client upstream should connect to static-server since it is passing - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName) + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "") // ########################### // ## with onlypassing=false @@ -235,7 +235,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { } _, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName, nil) serviceOptsV1 := &libservice.ServiceOpts{ Name: libservice.StaticServer2ServiceName, @@ -256,7 +256,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { } _, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName, nil) // Register static-server service resolver serviceResolver := &api.ServiceResolverConfigEntry{ @@ -318,7 +318,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { _, appPort := clientConnectProxy.GetAddr() _, adminPort := clientConnectProxy.GetAdminAddr() - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPort), "static-server-2-v2") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPort), "static-server-2-v2", "") libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server-2.default", "HEALTHY", 1) }, }, @@ -335,14 +335,19 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) { if oldVersionTmp.LessThan(libutils.Version_1_14) { buildOpts.InjectAutoEncryption = false } - cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) + cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: buildOpts, + ApplyDefaultProxySettings: true, + }) node := cluster.Agents[0] client := node.GetClient() staticClientProxy, staticServerProxy, err := createStaticClientAndServer(cluster) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) - libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName), nil) err = cluster.ConfigEntryWrite(&api.ProxyConfigEntry{ Kind: api.ProxyDefaults, diff --git a/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go b/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go index 5ccba9567739..f9407d600551 100644 --- a/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go +++ b/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go @@ -102,7 +102,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { require.NoError(t, clientSidecarService.Restart()) libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1) libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") } for _, tc := range tcs { diff --git a/test/integration/consul-container/test/upgrade/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering_http_test.go index f1d52777a7d6..3f7084fad47f 100644 --- a/test/integration/consul-container/test/upgrade/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering_http_test.go @@ -67,7 +67,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { if err != nil { return nil, nil, nil, err } - libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName) + libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName, nil) err = c.ConfigEntryWrite(&api.ProxyConfigEntry{ Kind: api.ProxyDefaults, @@ -100,7 +100,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { return serverConnectProxy, nil, func() {}, err }, extraAssertion: func(clientUpstreamPort int) { - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d/static-server-2", clientUpstreamPort), "static-server-2") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d/static-server-2", clientUpstreamPort), "static-server-2", "") }, }, { @@ -201,7 +201,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { GRPCPort: 8078, } _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts) - libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName) + libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName, nil) if err != nil { return nil, nil, nil, err } @@ -293,7 +293,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { GRPCPort: 8078, } _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts) - libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName) + libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName, nil) if err != nil { return nil, nil, nil, err } @@ -301,14 +301,14 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { _, appPorts := clientConnectProxy.GetAddrs() assertionFn := func() { // assert traffic can fail-over to static-server in peered cluster and restor to local static-server - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing", "") require.NoError(t, serverConnectProxy.Stop()) - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server", "") require.NoError(t, serverConnectProxy.Start()) - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing", "") // assert peer-static-server resolves to static-server in peered cluster - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[1]), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[1]), "static-server", "") } return serverConnectProxy, clientConnectProxy, assertionFn, nil }, @@ -376,7 +376,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { _, adminPort := clientSidecarService.GetAdminAddr() libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1) libassert.HTTPServiceEchoes(t, "localhost", port, "") - libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") // TODO: restart static-server-2's sidecar tc.extraAssertion(appPort) @@ -439,7 +439,11 @@ func createAndRegisterStaticClientSidecarWith2Upstreams(c *cluster.Cluster, dest } // Create a service and proxy instance - clientConnectProxy, err := libservice.NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", libservice.StaticClientServiceName), libservice.StaticClientServiceName, []int{cluster.ServiceUpstreamLocalBindPort, cluster.ServiceUpstreamLocalBindPort2}, node) + sidecarCfg := libservice.SidecarConfig{ + Name: fmt.Sprintf("%s-sidecar", libservice.StaticClientServiceName), + ServiceID: libservice.StaticClientServiceName, + } + clientConnectProxy, err := libservice.NewConnectService(context.Background(), sidecarCfg, []int{cluster.ServiceUpstreamLocalBindPort, cluster.ServiceUpstreamLocalBindPort2}, node) if err != nil { return nil, err } diff --git a/test/integration/consul-container/test/wanfed/wanfed_peering_test.go b/test/integration/consul-container/test/wanfed/wanfed_peering_test.go index 10b008745348..9a18a501b709 100644 --- a/test/integration/consul-container/test/wanfed/wanfed_peering_test.go +++ b/test/integration/consul-container/test/wanfed/wanfed_peering_test.go @@ -37,7 +37,11 @@ func TestPeering_WanFedSecondaryDC(t *testing.T) { t.Run("secondary dc can peer to alpha dc", func(t *testing.T) { // Create the gateway - _, err := libservice.NewGatewayService(context.Background(), "mesh", "mesh", c3.Servers()[0]) + gwCfg := libservice.GatewayConfig{ + Name: "mesh", + Kind: "mesh", + } + _, err := libservice.NewGatewayService(context.Background(), gwCfg, c3.Servers()[0]) require.NoError(t, err) // Create the peering connection From 9e88415e4de4571331fc9dff60d84c71f583074b Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Sun, 19 Mar 2023 16:33:52 -0700 Subject: [PATCH 112/421] Backport of Snapshot restore tests into release/1.15.x (#16672) * backport of commit a8525f831580580df029003d79c35a37d2e6fc15 * backport of commit a2d560517ca76e0848ce5cd16c73d54c0896cec1 * backport of commit 958c279dbcfac502d5dc9f9799a528aa5a41728a * backport of commit 1c1b88bc6a0060712a2f5942a2a474b400fd827b * backport of commit cd25976169f80a39a8dfc995b1d51fff213bdd3c * backport of commit d95619f85f0f05f45a20607b6dc613de354ac0dc * backport of commit 15a4645e7fd7e6d220a25b80eec16a13025db5f5 * backport of commit abb2203af5c88889373fa3bac9ce0fda2fe2e9e5 --------- Co-authored-by: Dhia Ayachi Co-authored-by: Paul Banks Co-authored-by: John Murret --- .changelog/16647.txt | 3 + go.mod | 10 +- go.sum | 21 ++-- .../consul-container/libs/cluster/builder.go | 24 +++++ .../libs/cluster/container.go | 10 +- .../libs/topology/peering_topology.go | 1 + .../test/snapshot/snapshot_restore_test.go | 97 +++++++++++++++++++ 7 files changed, 150 insertions(+), 16 deletions(-) create mode 100644 .changelog/16647.txt create mode 100644 test/integration/consul-container/test/snapshot/snapshot_restore_test.go diff --git a/.changelog/16647.txt b/.changelog/16647.txt new file mode 100644 index 000000000000..cbe38b3ed280 --- /dev/null +++ b/.changelog/16647.txt @@ -0,0 +1,3 @@ +```release-note:bug +raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. +``` diff --git a/go.mod b/go.mod index 2143ed05bb9e..0a525cbc8e50 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ exclude ( require ( github.com/NYTimes/gziphandler v1.0.1 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e - github.com/armon/go-metrics v0.3.10 + github.com/armon/go-metrics v0.4.1 github.com/armon/go-radix v1.0.0 github.com/aws/aws-sdk-go v1.42.34 github.com/coredns/coredns v1.6.6 @@ -62,10 +62,10 @@ require ( github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038 github.com/hashicorp/memberlist v0.5.0 - github.com/hashicorp/raft v1.3.11 + github.com/hashicorp/raft v1.4.0 github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.2 - github.com/hashicorp/raft-wal v0.2.4 + github.com/hashicorp/raft-wal v0.3.0 github.com/hashicorp/serf v0.10.1 github.com/hashicorp/vault/api v1.8.2 github.com/hashicorp/vault/api/auth/gcp v0.3.0 @@ -89,7 +89,7 @@ require ( github.com/rboyer/safeio v0.2.1 github.com/ryanuber/columnize v2.1.2+incompatible github.com/shirou/gopsutil/v3 v3.22.8 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -213,7 +213,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index 7234a8e0275a..461cd0136ff0 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -607,8 +607,9 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrq github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= +github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= +github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= @@ -616,8 +617,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qR github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc= github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw= github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY= -github.com/hashicorp/raft-wal v0.2.4 h1:Ke0ytMj8XyOVKQqFDmmgs/6hqkTJg0b/GO2a2XQBZ6A= -github.com/hashicorp/raft-wal v0.2.4/go.mod h1:JQ/4RbnKFi5Q/4rA73CekaYtHCJhU7qM7AQ4X5Y6q4M= +github.com/hashicorp/raft-wal v0.3.0 h1:Mi6RPoRbsxIIYZryI+bSTXHD97Ua6rIYO51ibYV9bkY= +github.com/hashicorp/raft-wal v0.3.0/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/vault/api v1.8.0/go.mod h1:uJrw6D3y9Rv7hhmS17JQC50jbPDAZdjZoTtrCCxxs7E= @@ -697,8 +698,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -921,8 +922,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -977,8 +978,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -987,8 +989,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 h1:8fDzz4GuVg4skjY2B0nMN7h6uN61EDVkuLyI2+qGHhI= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index 9ce5bf5c8c34..2807985e17e5 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -20,6 +20,13 @@ const ( ConsulCACertKey = "consul-agent-ca-key.pem" ) +type LogStore string + +const ( + LogStore_WAL LogStore = "wal" + LogStore_BoltDB LogStore = "boltdb" +) + // BuildContext provides a reusable object meant to share common configuration settings // between agent configuration builders. type BuildContext struct { @@ -41,6 +48,7 @@ type BuildContext struct { tlsCertIndex int // keeps track of the certificates issued for naming purposes aclEnabled bool + logStore LogStore } func (c *BuildContext) DockerImage() string { @@ -89,6 +97,9 @@ type BuildOptions struct { // ACLEnabled configures acl in agent configuration ACLEnabled bool + + //StoreLog define which LogStore to use + LogStore LogStore } func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { @@ -103,6 +114,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { useAPIWithTLS: opts.UseAPIWithTLS, useGRPCWithTLS: opts.UseGRPCWithTLS, aclEnabled: opts.ACLEnabled, + logStore: opts.LogStore, } if ctx.consulImageName == "" { @@ -202,6 +214,18 @@ func NewConfigBuilder(ctx *BuildContext) *Builder { b.conf.Set("acl.enable_token_persistence", true) } + ls := string(ctx.logStore) + if ls != "" && (ctx.consulVersion == "local" || + semver.Compare("v"+ctx.consulVersion, "v1.15.0") >= 0) { + // Enable logstore backend for version after v1.15.0 + if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) { + ls = string(LogStore_BoltDB) + } + b.conf.Set("raft_logstore.backend", ls) + } else { + b.conf.Unset("raft_logstore.backend") + } + return b } diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 3ad1b2d35169..4e3e1c39901c 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -154,11 +154,17 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) if httpPort > 0 { - uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + for i := 0; i < 10; i++ { + uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + if err != nil { + time.Sleep(500 * time.Millisecond) + continue + } + clientAddr = uri + } if err != nil { return nil, err } - clientAddr = uri } else if httpsPort > 0 { uri, err := podContainer.PortEndpoint(ctx, "8501", "https") diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 0e853f7adda3..3e7ac8d067b0 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -198,6 +198,7 @@ func NewCluster( AllowHTTPAnyway: true, ConsulVersion: config.BuildOpts.ConsulVersion, ACLEnabled: config.BuildOpts.ACLEnabled, + LogStore: config.BuildOpts.LogStore, } ctx := libcluster.NewBuildContext(t, opts) diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go new file mode 100644 index 000000000000..1d82b1cfb10c --- /dev/null +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -0,0 +1,97 @@ +package snapshot + +import ( + "fmt" + "github.com/hashicorp/consul/api" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/stretchr/testify/require" + "io" + "testing" +) + +func TestSnapshotRestore(t *testing.T) { + + cases := []libcluster.LogStore{libcluster.LogStore_WAL, libcluster.LogStore_BoltDB} + + for _, c := range cases { + t.Run(fmt.Sprintf("test log store: %s", c), func(t *testing.T) { + testSnapShotRestoreForLogStore(t, c) + }) + } +} + +func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) { + + const ( + numServers = 3 + ) + + // Create initial cluster + cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, + LogStore: logStore, + }, + ApplyDefaultProxySettings: true, + }) + + client := cluster.APIClient(0) + libcluster.WaitForLeader(t, cluster, client) + libcluster.WaitForMembers(t, client, 3) + + for i := 0; i < 100; i++ { + _, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil) + require.NoError(t, err) + } + + var snapshot io.ReadCloser + var err error + snapshot, _, err = client.Snapshot().Save(nil) + require.NoError(t, err) + + err = cluster.Terminate() + require.NoError(t, err) + // Create a fresh cluster from scratch + cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, + LogStore: logStore, + }, + ApplyDefaultProxySettings: true, + }) + client2 := cluster2.APIClient(0) + + libcluster.WaitForLeader(t, cluster2, client2) + libcluster.WaitForMembers(t, client2, 3) + + // Restore the saved snapshot + require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) + + libcluster.WaitForLeader(t, cluster2, client2) + + followers, err := cluster2.Followers() + require.NoError(t, err) + require.Len(t, followers, 2) + + // use a follower api client and set `AllowStale` to true + // to test the follower snapshot install code path as well. + fc := followers[0].GetClient() + + for i := 0; i < 100; i++ { + kv, _, err := fc.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true}) + require.NoError(t, err) + require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i)) + require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i))) + } + +} From 70b8c6b4dfd46e393a81262272ae942b0d33719c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Mon, 20 Mar 2023 12:57:13 -0400 Subject: [PATCH 113/421] Backport/squashing fix for enterprise meta (#16651) * Backport squash structures for config entries * Add changelog * Rename changelog file for current PR * change changelog file name * Add enterprise only tag to changelog * Fix changelog --- .changelog/16651.txt | 3 +++ agent/structs/config_entry_routes.go | 5 +++-- agent/structs/config_entry_status.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 .changelog/16651.txt diff --git a/.changelog/16651.txt b/.changelog/16651.txt new file mode 100644 index 000000000000..c297cca489d9 --- /dev/null +++ b/.changelog/16651.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal. +``` diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 1235723f89f2..683ec9f3fac0 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -4,8 +4,9 @@ import ( "fmt" "strings" - "github.com/hashicorp/consul/acl" "github.com/miekg/dns" + + "github.com/hashicorp/consul/acl" ) // BoundRoute indicates a route that has parent gateways which @@ -443,7 +444,7 @@ type HTTPService struct { // to routing it to the upstream service Filters HTTPFilters - acl.EnterpriseMeta + acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` } func (s HTTPService) ServiceName() ServiceName { diff --git a/agent/structs/config_entry_status.go b/agent/structs/config_entry_status.go index 85140f3e5aae..2a56ba08a016 100644 --- a/agent/structs/config_entry_status.go +++ b/agent/structs/config_entry_status.go @@ -22,7 +22,7 @@ type ResourceReference struct { // unused, this should be blank. SectionName string - acl.EnterpriseMeta + acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` } func (r *ResourceReference) String() string { From 459b42dc5cad3103d1419b448ecb9235d15c4ebf Mon Sep 17 00:00:00 2001 From: Nitya Dhanushkodi Date: Mon, 20 Mar 2023 10:01:04 -0700 Subject: [PATCH 114/421] [release/1.15.x] peering: peering partition failover fixes (#16675) add local source partition for peered upstreams --- .changelog/16675.txt | 3 +++ agent/proxycfg/upstreams.go | 3 +++ agent/xds/clusters.go | 9 ++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .changelog/16675.txt diff --git a/.changelog/16675.txt b/.changelog/16675.txt new file mode 100644 index 000000000000..f72eedc61c82 --- /dev/null +++ b/.changelog/16675.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fixes a bug where the importing partition was not added to peered failover targets, which causes issues when the importing partition is a non-default partition. +``` diff --git a/agent/proxycfg/upstreams.go b/agent/proxycfg/upstreams.go index 367eeda62cc8..23fab4d2c0f5 100644 --- a/agent/proxycfg/upstreams.go +++ b/agent/proxycfg/upstreams.go @@ -470,6 +470,9 @@ func (s *handlerUpstreams) watchUpstreamTarget(ctx context.Context, snap *Config if opts.peer != "" { uid = NewUpstreamIDFromTargetID(opts.chainID) + // chainID has the partition stripped. However, when a target is in a cluster peer, the partition should be set + // to the local partition (i.e chain.Partition), since the peered target is imported into the local partition. + uid.OverridePartition(opts.entMeta.PartitionOrDefault()) correlationID = upstreamPeerWatchIDPrefix + uid.String() } diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 7ee9a5577dcc..80526c65e51f 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -976,7 +976,7 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService( // entire cluster. outlierDetection.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: 100} - s.Logger.Trace("generating cluster for", "cluster", clusterName) + s.Logger.Trace("generating cluster for", "cluster", clusterName, "uid", uid) if c == nil { c = &envoy_cluster_v3.Cluster{ Name: clusterName, @@ -1044,10 +1044,13 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService( makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), ) err = injectSANMatcher(commonTLSContext, peerMeta.SpiffeID...) + s.Logger.Trace("injecting SAN matcher rules for cluster %q with SPIFFE IDs: %+v", clusterName, peerMeta.SpiffeID) + if err != nil { return nil, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", clusterName, err) } + s.Logger.Trace("injecting TLS context for cluster %q with SNI: %+v", clusterName, peerMeta.PrimarySNI()) tlsContext := &envoy_tls_v3.UpstreamTlsContext{ CommonTlsContext: commonTLSContext, Sni: peerMeta.PrimarySNI(), @@ -1305,6 +1308,10 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( targetUID := proxycfg.NewUpstreamIDFromTargetID(targetData.targetID) if targetUID.Peer != "" { + // targetID already has a stripped partition, so targetUID will not have a partition either. However, + // when a failover target is in a cluster peer, the partition should be set to the local partition (i.e + // chain.Partition), since that's where the data is imported to. + targetUID.OverridePartition(chain.Partition) peerMeta, found := upstreamsSnapshot.UpstreamPeerMeta(targetUID) if !found { s.Logger.Warn("failed to fetch upstream peering metadata for cluster", "target", targetUID) From 7f7d282a343f01751dd7ce1d361e5d51e908c392 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 20 Mar 2023 11:14:05 -0700 Subject: [PATCH 115/421] Backport of Add validation for apigw creation with no routes into release/1.15.x (#16679) --- .changelog/16649.txt | 3 +++ agent/structs/config_entry_discoverychain_test.go | 13 +++++++++++-- agent/structs/config_entry_gateways.go | 3 +++ agent/structs/config_entry_gateways_test.go | 7 +++++++ website/content/commands/config/delete.mdx | 9 +++++---- website/content/commands/config/list.mdx | 1 + website/content/commands/config/read.mdx | 1 + website/content/commands/config/write.mdx | 1 + 8 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 .changelog/16649.txt diff --git a/.changelog/16649.txt b/.changelog/16649.txt new file mode 100644 index 000000000000..e510558ff907 --- /dev/null +++ b/.changelog/16649.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Adds validation to ensure the API Gateway has a listener defined when created +``` \ No newline at end of file diff --git a/agent/structs/config_entry_discoverychain_test.go b/agent/structs/config_entry_discoverychain_test.go index b16f48f63b4d..db4ac425ebe9 100644 --- a/agent/structs/config_entry_discoverychain_test.go +++ b/agent/structs/config_entry_discoverychain_test.go @@ -410,8 +410,17 @@ func TestConfigEntries_ListRelatedServices_AndACLs(t *testing.T) { }, }, { - name: "api-gateway", - entry: &APIGatewayConfigEntry{Name: "test"}, + name: "api-gateway", + entry: &APIGatewayConfigEntry{ + Name: "test", + Listeners: []APIGatewayListener{ + { + Name: "test", + Port: 100, + Protocol: "http", + }, + }, + }, expectACLs: []testACL{ { name: "no-authz", diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 885a301fc467..5309af35ad3d 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -769,6 +769,9 @@ func (e *APIGatewayConfigEntry) Validate() error { return err } + if len(e.Listeners) == 0 { + return fmt.Errorf("api gateway must have at least one listener") + } if err := e.validateListenerNames(); err != nil { return err } diff --git a/agent/structs/config_entry_gateways_test.go b/agent/structs/config_entry_gateways_test.go index ca68ea4f40a5..1302cb2ad7ad 100644 --- a/agent/structs/config_entry_gateways_test.go +++ b/agent/structs/config_entry_gateways_test.go @@ -1126,6 +1126,13 @@ func TestGatewayService_Addresses(t *testing.T) { func TestAPIGateway_Listeners(t *testing.T) { cases := map[string]configEntryTestcase{ + "no listeners defined": { + entry: &APIGatewayConfigEntry{ + Kind: "api-gateway", + Name: "api-gw-one", + }, + validateErr: "api gateway must have at least one listener", + }, "listener name conflict": { entry: &APIGatewayConfigEntry{ Kind: "api-gateway", diff --git a/website/content/commands/config/delete.mdx b/website/content/commands/config/delete.mdx index 3fc9e6618b08..134d6885e666 100644 --- a/website/content/commands/config/delete.mdx +++ b/website/content/commands/config/delete.mdx @@ -27,6 +27,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ------------------ | +| api-gateway | `mesh:write` | | ingress-gateway | `operator:write` | | proxy-defaults | `operator:write` | | service-defaults | `service:write` | @@ -45,16 +46,16 @@ Usage: `consul config delete [options]` - `-kind` - Specifies the kind of the config entry to read. - `-name` - Specifies the name of the config entry to delete. The name of the - `proxy-defaults` config entry must be `global`, and the name of the `mesh` - config entry must be `mesh`. +`proxy-defaults` config entry must be `global`, and the name of the `mesh` +config entry must be `mesh`. - `-filename` - Specifies the file describing the config entry to delete. - `-cas` - Perform a Check-And-Set operation. Specifying this value also - requires the -modify-index flag to be set. The default value is false. +requires the -modify-index flag to be set. The default value is false. - `-modify-index=` - Unsigned integer representing the ModifyIndex of the - config entry. This is used in combination with the -cas flag. +config entry. This is used in combination with the -cas flag. #### Enterprise Options diff --git a/website/content/commands/config/list.mdx b/website/content/commands/config/list.mdx index c72e3e903de4..1a70af178725 100644 --- a/website/content/commands/config/list.mdx +++ b/website/content/commands/config/list.mdx @@ -27,6 +27,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ----------------- | +| api-gateway | `mesh:read` | | ingress-gateway | `service:read` | | proxy-defaults | `` | | service-defaults | `service:read` | diff --git a/website/content/commands/config/read.mdx b/website/content/commands/config/read.mdx index a50574aaed7e..7a49482c5b3f 100644 --- a/website/content/commands/config/read.mdx +++ b/website/content/commands/config/read.mdx @@ -28,6 +28,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ----------------- | +| api-gateway | `mesh:read` | | ingress-gateway | `service:read` | | proxy-defaults | `` | | service-defaults | `service:read` | diff --git a/website/content/commands/config/write.mdx b/website/content/commands/config/write.mdx index 7e586aff759a..24e17aa34ae4 100644 --- a/website/content/commands/config/write.mdx +++ b/website/content/commands/config/write.mdx @@ -30,6 +30,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ------------------ | +| api-gateway | `mesh:write` | | ingress-gateway | `operator:write` | | proxy-defaults | `operator:write` | | service-defaults | `service:write` | From 1007444ac13fefaf6e93fab1d4da5f01ebeed1b6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 20 Mar 2023 12:09:54 -0700 Subject: [PATCH 116/421] Backport of Fix route subscription when using namespaces into release/1.15.x (#16680) * backport of commit 312492bdf7556187d9fac587381da1a648bc101e * backport of commit 099340902cb0db65c282dd42552b74370e2d1e59 * backport of commit 2fcae963d99d8f888bad6d3c704bd0bbbdf0ea8e --------- Co-authored-by: jm96441n --- .changelog/_16677.txt | 3 +++ agent/proxycfg/api_gateway.go | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 .changelog/_16677.txt diff --git a/.changelog/_16677.txt b/.changelog/_16677.txt new file mode 100644 index 000000000000..0bf621f09ac4 --- /dev/null +++ b/.changelog/_16677.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where routes defined in a different namespace than a gateway would fail to register. [[GH-16677](https://github.com/hashicorp/consul/pull/16677)]. +``` diff --git a/agent/proxycfg/api_gateway.go b/agent/proxycfg/api_gateway.go index c9b5ad1007e8..ad0d5f203dc4 100644 --- a/agent/proxycfg/api_gateway.go +++ b/agent/proxycfg/api_gateway.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/hashicorp/consul/acl" cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/proxycfg/internal/watch" "github.com/hashicorp/consul/agent/structs" @@ -45,13 +46,13 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err } // Watch the api-gateway's config entry - err = h.subscribeToConfigEntry(ctx, structs.APIGateway, h.service, gatewayConfigWatchID) + err = h.subscribeToConfigEntry(ctx, structs.APIGateway, h.service, h.proxyID.EnterpriseMeta, gatewayConfigWatchID) if err != nil { return snap, err } // Watch the bound-api-gateway's config entry - err = h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, gatewayConfigWatchID) + err = h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, h.proxyID.EnterpriseMeta, gatewayConfigWatchID) if err != nil { return snap, err } @@ -80,13 +81,13 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err return snap, nil } -func (h *handlerAPIGateway) subscribeToConfigEntry(ctx context.Context, kind, name, watchID string) error { +func (h *handlerAPIGateway) subscribeToConfigEntry(ctx context.Context, kind, name string, entMeta acl.EnterpriseMeta, watchID string) error { return h.dataSources.ConfigEntry.Notify(ctx, &structs.ConfigEntryQuery{ Kind: kind, Name: name, Datacenter: h.source.Datacenter, QueryOptions: structs.QueryOptions{Token: h.token}, - EnterpriseMeta: h.proxyID.EnterpriseMeta, + EnterpriseMeta: entMeta, }, watchID, h.ch) } @@ -172,7 +173,7 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd return fmt.Errorf("unexpected route kind on gateway: %s", ref.Kind) } - err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, routeConfigWatchID) + err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, routeConfigWatchID) if err != nil { // TODO May want to continue return err @@ -185,7 +186,7 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd seenRefs[ref] = struct{}{} snap.APIGateway.Certificates.InitWatch(ref, cancel) - err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, inlineCertificateConfigWatchID) + err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID) if err != nil { // TODO May want to continue return err @@ -391,7 +392,7 @@ func (h *handlerAPIGateway) handleRouteConfigUpdate(ctx context.Context, u Updat snap.APIGateway.Upstreams.set(ref, listener, set) } snap.APIGateway.UpstreamsSet.set(ref, seenUpstreamIDs) - //snap.APIGateway.Hosts = TODO + // snap.APIGateway.Hosts = TODO snap.APIGateway.AreHostsSet = true // Stop watching any upstreams and discovery chains that have become irrelevant From 9056b87ad980335f5fb501091b5c97318abafce3 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 20 Mar 2023 15:53:55 -0700 Subject: [PATCH 117/421] backport of commit e3b6545a6e6ea801d4f190eb8be864a4f60b860b (#16699) --- .../content/api-docs/connect/intentions.mdx | 20 +++++++++---------- website/content/commands/intention/check.mdx | 2 +- website/content/commands/intention/create.mdx | 2 +- website/content/commands/intention/delete.mdx | 2 +- website/content/commands/intention/get.mdx | 2 +- website/content/commands/intention/list.mdx | 2 +- website/content/commands/intention/match.mdx | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/website/content/api-docs/connect/intentions.mdx b/website/content/api-docs/connect/intentions.mdx index 870b6d4b77d9..f5b38e082cf4 100644 --- a/website/content/api-docs/connect/intentions.mdx +++ b/website/content/api-docs/connect/intentions.mdx @@ -48,7 +48,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -154,7 +154,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -251,7 +251,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -305,7 +305,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -377,7 +377,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -440,7 +440,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -527,7 +527,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -582,7 +582,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -638,7 +638,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. @@ -698,7 +698,7 @@ The table below shows this endpoint's support for

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/check.mdx b/website/content/commands/intention/check.mdx index a0c384c6461c..d9a4d5d328c3 100644 --- a/website/content/commands/intention/check.mdx +++ b/website/content/commands/intention/check.mdx @@ -36,7 +36,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/create.mdx b/website/content/commands/intention/create.mdx index b41b9c502fb4..31f2ae899a9b 100644 --- a/website/content/commands/intention/create.mdx +++ b/website/content/commands/intention/create.mdx @@ -30,7 +30,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/delete.mdx b/website/content/commands/intention/delete.mdx index 1f6971c49062..87d9d65e9051 100644 --- a/website/content/commands/intention/delete.mdx +++ b/website/content/commands/intention/delete.mdx @@ -24,7 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/get.mdx b/website/content/commands/intention/get.mdx index 6ac253c898ec..68dd744404f6 100644 --- a/website/content/commands/intention/get.mdx +++ b/website/content/commands/intention/get.mdx @@ -29,7 +29,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/list.mdx b/website/content/commands/intention/list.mdx index cc5130ac993d..928f416095d8 100644 --- a/website/content/commands/intention/list.mdx +++ b/website/content/commands/intention/list.mdx @@ -24,7 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. diff --git a/website/content/commands/intention/match.mdx b/website/content/commands/intention/match.mdx index 4936e12ba358..e0bbfc1222e5 100644 --- a/website/content/commands/intention/match.mdx +++ b/website/content/commands/intention/match.mdx @@ -29,7 +29,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint

    1 Intention ACL rules are specified as part of a{' '} service rule. See{' '} - + Intention Management Permissions {' '} for more details. From 2b07d8010bc64b00b2e4d90de4a7791efe653d4b Mon Sep 17 00:00:00 2001 From: Anita Akaeze Date: Tue, 21 Mar 2023 09:50:23 -0400 Subject: [PATCH 118/421] NET-2397: Add readme.md to upgrade test subdirectory (#16610) (#16696) * NET-2397: Add readme.md to upgrade test subdirectory * remove test code * fix link and update steps of adding new test cases (#16654) * fix link and update steps of adding new test cases * Apply suggestions from code review --------- --------- Co-authored-by: cskh Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> --- docs/README.md | 1 + .../consul-container/test/upgrade/README.md | 179 +++++++++++++++++- .../test/util/upgrade_tests_workflow.png | Bin 0 -> 511431 bytes 3 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 test/integration/consul-container/test/util/upgrade_tests_workflow.png diff --git a/docs/README.md b/docs/README.md index 0c24ff8f9920..8f0be780011f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,6 +38,7 @@ Also see the [FAQ](./faq.md). ## Other Docs 1. [Integration Tests](../test/integration/connect/envoy/README.md) +1. [Upgrade Tests](../test/integration/consul-container/test/upgrade/README.md) ## Important Directories diff --git a/test/integration/consul-container/test/upgrade/README.md b/test/integration/consul-container/test/upgrade/README.md index adffc5344ad3..6815323af0df 100644 --- a/test/integration/consul-container/test/upgrade/README.md +++ b/test/integration/consul-container/test/upgrade/README.md @@ -1,17 +1,186 @@ -# Consul Upgrade Integration tests +# Upgrade Integration Tests -## Local run +- [Introduction](#introduction) + - [How it works](#how-it-works) +- [Getting Started](#getting-started) + - [Prerequisites](#prerequisites) + - [Running Upgrade integration tests](#running-upgrade-integration-tests) +- [Adding a new upgrade integration test](#adding-a-new-upgrade-integration-test) + - [Errors Test Cases](#errors-test-cases) +- [FAQS](#faqs) + + +## Introduction + +The goal of upgrade tests is to ensure problem-free upgrades on supported upgrade paths. At any given time, Consul supports the latest minor release, and two older minor releases, e.g. 1.15, 1.14, and 1.13. Upgrades to any higher version are permitted, including skipping a minor version e.g. from 1.13 to 1.15. + +The upgrade tests also aims to highlight errors that may occur as users attempt to upgrade their current version to a newer version. + +### How it works + +This diagram illustrates the deployment architecture of an upgrade test, where +two consul agents (one server and one client), a static-server, static-client, +and envoy sidecars are deployed. + +isolated + +> Note that all consul agents and user workloads such as application services, mesh-gateway are running in docker containers. + +In general, each upgrade test has following steps: +1. Create a cluster with a specified number of server and client agents, then enable the feature to be tested. +2. Create some workload in the cluster, e.g., registering 2 services: static-server, static-client. +Static-server is a simple http application and the upstream service of static-client. +3. Make additional configuration to the cluster. For example, configure Consul intention to deny +connection between static client and server. Ensure that a connection cannot be made. +4. Upgrade Consul cluster to the `target-version` and restart the Envoy sidecars +(we restart Envoy sidecar to ensure the upgraded Consul binary can read the state from +the previous version and generate the correct Envoy configurations) +5. Re-validate the client, server and sidecars to ensure the persisted data from the pervious +version can be accessed in the target version. Verify connection / disconnection +(e.g., deny Action) + +## Getting Started +### Prerequisites +To run the upgrade test, the following tools are required: +- install [Go](https://go.dev/) (the version should match that of our CI config's Go image). +- install [`golangci-lint`](https://golangci-lint.run/usage/install/) +- install [`Makefile`](https://www.gnu.org/software/make/manual/make.html) +- [`Docker`](https://docs.docker.com/get-docker/) required to run tests locally + +### Running Upgrade integration tests - run `make dev-docker` -- run the tests, e.g., `go test -run ^TestBasicConnectService$ ./test/basic -v` +- run the single test `go test -v -timeout 30m -run ^TestACL_Upgrade_Node_Token$ ./.../upgrade/` +- run all upgrade tests `go test -v -timeout 30m -run ./.../upgrade` -To specify targets and latest image pass `target-version` and `latest-version` +To specify targets and latest image pass `--target-version` and `--latest-version` to the tests. By default, it uses the `consul` docker image with respectively `local` and `latest` tags. To use dev consul image, pass `target-image` and `target-version`: - -target-image hashicorppreview/consul -target-version 1.14-dev + -target-image hashicorppreview/consul -target-version 1.15-dev By default, all container's logs are written to either `stdout`, or `stderr`; this makes it hard to debug, when the test case creates many containers. To disable following container logs, run the test with `-follow-log false`. + +Below are the supported CLI options +| Flags | Default value | Description | +| ----------- | ----------- | ----------- | +| --latest-image | `consul` in OSS, `hashicorp/consulenterprise` in ENT | Name of the Docker image to deploy initially. +| --latest-version | latest | Tag of the Docker image to deploy initially. +| --target-image | `consul` in OSS, `hashicorp/consulenterprise` in ENT | Name of the Docker image to upgrade to. +| --target-version | local | Tag of the Docker image to upgrade to. `local` is the tag built by `make dev-docker` above. +| -follow-log | true | Emit all container logs. These can be noisy, so we recommend `--follow-log=false` for local development. + + +## Adding a new upgrade integration test + +All upgrade tests are defined in [test/integration/consul-container/test/upgrade](/test/integration/consul-container/test/upgrade) subdirectory. The test framework uses +[functional table-driven tests in Go](https://yourbasic.org/golang/table-driven-unit-test/) and +using function types to modify the basic configuration for each test case. + +Following is a guide for adding a new upgrade test case. +1. Create consul cluster(s) with a specified version. Some utility functions are provided to make +a single cluster or two peered clusters: + +```go + // NewCluster creates a single cluster + cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulVersion: oldVersion, + }, + }) + + // BasicPeeringTwoClustersSetup creates two peered clusters, named accpeting and dialing + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, oldVersion, false) +``` + +2. For tests with multiple test cases, it should always start by invoking +```go + type testcase struct { + name string + create func() + extraAssertion func() + } +``` +see example [here](./l7_traffic_management/resolver_default_subset_test.go). For upgrade tests with a single test case, they can be written like +```go + run := func(t *testing.T, oldVersion, targetVersion string) { + // insert test + } + t.Run(fmt.Sprintf("Upgrade from %s to %s", utils.LatestVersion, utils.TargetVersion), + func(t *testing.T) { + run(t, utils.LatestVersion, utils.TargetVersion) + }) +``` +see example [here](./acl_node_test.go) + +Addtitional configurations or user-workload can be created with a customized [`create` function](./l7_traffic_management/resolver_default_subset_test.go). + +3. Call the upgrade method and assert the upgrading cluster succeeds. +We also restart the envoy proxy to make sure the upgraded agent can generate +the correct envoy configurations. + +```go + err = cluster.StandardUpgrade(t, context.Background(), targetVersion) + require.NoError(t, err) + require.NoError(t, staticServerConnectProxy.Restart()) + require.NoError(t, staticClientConnectProxy.Restart()) +``` + +4. Verify the user workload after upgrade, e.g., + +```go + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPort), "static-server-2-v2", "") +``` + +### Errors Test Cases +There are some caveats for special error handling of versions prior to `1.14`. +Upgrade tests for features such peering, had API changes that returns an error if attempt to upgrade, and should be accounted for in upgrade tests. If running upgrade tests for any version before `1.14`, the following lines of code needs to be added to skip test or it will not pass. + +```go + fromVersion, err := version.NewVersion(utils.LatestVersion) + require.NoError(t, err) + if fromVersion.LessThan(utils.Version_1_14) { + continue + } +``` +See example [here](https://github.com/hashicorp/consul-enterprise/blob/005a0a92c5f39804cef4ad5c4cd6fd3334b95aa2/test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go#L92-L96) + +To write tests for bugs found during upgrades, see example on how to add a testcase for those scenarios [here](./fullstopupgrade_test.go). + +## FAQS + +**Q.** Are containers' ports (e.g., consul's 8500, envoy sidecar's admin port +or local upstream port) exposed on the docker host? \ +**A.** Yes, they are exposed. However, they are exposed through a [pod container](https://github.com/hashicorp/consul/blob/57e034b74621180861226a01efeb3e9cedc74d3a/test/integration/consul-container/libs/cluster/container.go#L132). +That is, a consul agent and the envoy proxy containers registered with the agent +share the [same Linux network namespace (i.e., they share `localhost`)](https://github.com/hashicorp/consul/blob/57e034b74621180861226a01efeb3e9cedc74d3a/test/integration/consul-container/libs/cluster/app.go#L23-L30) as the pod container. +The pod container use the same prefix as the consul agent in its name. + +**Q.** To troubleshoot, how can I send API request or consul command to the deployed cluster? \ +**A.** To send an API request or command to the deployed cluster, ensure that a cluster, services and sidecars have been created. See example below: +```go + cluster, _, _ := topology.NewCluster() + clientService := createServices(t, cluster) + _, port := clientService.GetAddr() + _, adminPort := clientService.GetAdminAddr() + ... + time.Sleep(900 * time.Second) + fmt.Println(port, adminPort) +``` +Then in your terminal `docker ps -a | grep consul` to get the running services and cluster. Exec in the cluster and run commands directly or make API request to `localhost:port` to relevant service or `localhost:adminPort` for envoy. + +**Q.** To troubleshoot, how can I access the envoy admin page? \ +**A.** To access envoy admin page, ensure that a cluster, services and sidecars have been created. Then get the adminPort for the client or server sidecar. See example on how to get the port above. Then navigate to a browser and go to the url `http://localhost:adminPort/` + +**Q.** My test is stuck with the error "could not start or join all agents: container 0: port not found"? \ +**A.** Simply re-run the tests. If the error persists, prune docker images `docker system prune`, run `make dev-docker`, then re-run tests again. + +**Q.** How to clean up the resources created the upgrade test? +**A.** Run the command `docker ps | grep consul` to find all left over resources, then `docker stop {CONTAINER_ID} && docker rm {CONTAINER_ID}` diff --git a/test/integration/consul-container/test/util/upgrade_tests_workflow.png b/test/integration/consul-container/test/util/upgrade_tests_workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..f8403259c36d85d85b744f08ba1662bd857c767a GIT binary patch literal 511431 zcmeFZby$>N_cscHqI7qIfHcxMbSMpy(j7`GB|~?Y(k&n*p#lO!m(n09Ej1wB6Xza% zp5ODm&ih{PALpNQu5&%ZHO_tCd+)XOtiAVIpA}<_wx$XmHZ?XH8XBIOs-i9$8s-x; zG>oA8_kbsNuz$~>p`jZ&DJW>GDJU>%d%D>>x!R$jaiqLR)_QtJhqQ03)OwDRiU;TU zvp6;RgoW%7CHXWb{U*o2Y-)s`G`l><4og$gD;`so`#N7m4nmn7SFrB0a%BDX;{4?A zE)K%qHsi(G*%cP!lR!@&ey`eV4|jpm1YP3L9(9KWUEv!1A~H+T!(3T_Qxj;HE^7^% z&Du<W^AMt>Ly1v8Ba|o+fAvBR$CNsD3THSMD z>ze^vRy6Sn5y8i{6s91yACq)JwP8>Dq%VBJ+RR5X^|(sIgnFV`?t!)hqA;B(lN_T06?~^7g(6hl~lY4X&GkwwR_%REKumsddQl*i<>kek@_sdIDzZ?Xqf2KXm^1tbl@$6PV+z4%IF+u7=NDMK|_mjLc{!5 zA5GvB^-2cbs4;&(F;XMZuz)*a;O$#@=ij|CpA=&J`x+w%XhW0JQ&3X_KJ{!o?d)E9 zIl6gE<&)w94<5Ly8hfFkkv~Me(baTWj{y3!PWndPMjGl8Hf}F?t!&+_?Rb4(xTENx zN%=|umoMzRtr&e@xW4p~@Rer%(?bHdMm^?ZX8hB|+eMn$NJE=Z!Ohc-QG}PDm!DY% zn~{-G%G1_fLRV4wU&DbrX=X=nZ+8hkzSpl`^S&13b@O!K6A%{{=i`6O_xLdn(1XXz z@1?huFV9OamcK;)O-Iqr%f{2m-P_6SB_oQim9?9Xw=^>|iqZeP{?60R*Xci;UV8oO zYXM)757ojaz{}6~KeU0NQmCgA+D^W9uEvT^F94YVJY>Ygg{1!U|6f}E!|@*@js7!I zfL~nfpF{uA^nV}v%*)PG!R-aW(_7{r>-CrKKO6rwP>K)r)&C)izjFTb6cDrwwiMt0 zESe0qcIW}H#Avimidy=>C$P&Y!RR@5`*I$@Xs+PqoG%`J#K##)h<9znIdW^ZUampak1G zbs*G5l8MICJ|WIIXp%X22@Xwqy38?O=9Txr^Amqc9)c9zRCzdkf6o46A6wo6GB?H^ zlVq`v>!Bj8VGom#(>s4Uq?wc?QUcjVY|?K)_%+bjTdET`0QTq(ew5WLL-<4b_E*=_ z=jBkX*@ChU2G*kHANZk9$g`8Ph1A>4k^AfO|8CR&pSD_1d~lD|yZEsnc)K^0jkEa6 z(7?zjboG;t)IIfI1T&bD^O5yTGn#D}a(17PiQw%5hW9X|j2_+;3HzWBINfyLrFQ3(Tz53!0Etw>9L&kQ>sY)P- z(xqM_4hmlS!}6}193=}m&5lt72t&fs`2`ilUIVNef12&xES_H;P2IeZtMu**K@XjGxI5a8m>bx+b6O|zWUtnV`^+e^pH911aG%tuVTt%bqTuUy;R!uM z=+@p#Utb**)Nf538&XOjld+`{V)P;`CD5b~ikp%BPi9w)MOhdHx#Km)K*F3cm#{_D z!d}PN2!5+*Q%?A)>W`4S8)MwkKkA$L)*Q*@;l6iW&f2Gln$UKm46>KzVK$a`?0Q{D z4X0S+HU8YBEz`#hU7S?R)gN{cH$XM2 zt4D6VV#z*Qm}IxDY2{|~%{ZI8*TCxw8D0D+81utMLFHi6?lFWx%0cW~vT5|2uspG_ zC%u7yZ+L%jb9xXxb_lJ1t zWKv3><8v!3)2NHQzsxeU>3EX7HGHKDrRrl&s*tcPxL@gx z5$P1 zN#KWJp0;Y~MD)S`p|NezTXd>5R7h(~Y!{BTiW|VWgl@W!wmNZ5;jshLd(5GdsYtbk zF4mKe{kk*rg4Nd>p+qZzOvfEgd8ni8i}oKDd0?WbQPBUeNIu4~mKx=Vz?Bbz;IHm{ zu`2c25*ndDSk6?|tLMksW3X^S(t%5@el)*%zb__O1FUTUeY*RIEZLT90L%vty;(W6 z7=uV7FHpM^j=je1z{}+ipMln0Yo792*%$9CUJ-pW2cO?S^~}I@f#>#{m8of0$2PkM zCnh<(nwLFWCBmKa32!ez@?gr&#osPIu)M(S$uGdSodHzeSPND`EyN5ge0~PgvrkhT zz)d|1)!y0Q50plp!NPiXuD@(sly-T<$a<=}a~zz5_MjfNj5{~b`EB)Ne}s;uj3Rub zZkeNjB}i~M#Y>RMAvmK9>t;d(lbq)L&j_99P?Ohoc*E)nTxG5k^ZU8u>ko6yL$nTVi4703B!hA@X9UQI4PKP+S zD_WJ?P9fgtO}BFYvYn2t_Msw6Wv~PVhBoDsAhZB%D1O|mTRlCDH!oEV4Ax_^;4F_< zK|!Cvh+2?1X)8j}&ZsKtQP22%z8)uQWwAXyQxXc+B;%NYJvjo2pkohV6Ev{?%#aQS(naO%^@+pX47K*b=d)41Nfd zHJ%E>)0@E_GD<;>sX9To)3_B@f5ydXqeKpHs2;{j7e1KffA;@VBFw*j8oJ51aq_g8 z@fnl{I4=-K>IG$gF-#Rs4(38iinJ|GPvtB~8qtc_zZXmSw1$oorw-~iyHX1o2-{AkDT!jX z5<7=^g>l39`o)TQg|S}bB0dk$#(eO^2Z8!a$bOF1f6zap zX+YJdS{!qBr2vd+MR@=54^)JU$Rka3UqAC==Z?2#oltz0*@_7CJm3CK^U&k)hpGN= zhZegpvso|X-JEf9Y&i`K`t+Dml9$cdGX^k4KSgLfCgZAFuS`}|b*oDoK42&s{9+x< z^%ASAsycQ|l^SRFz^58|cXXOuWJtfdI<{P%nW?>9lk*a;!f}Hbb0rcPQcoZL;LFKM zJh=!h_1tEVpSTunRGMQI)ZzRO-8}GprR|mP_Vb)={ic=8!_^~BYwQSY4I{Wm!KIOUE#=@J=n(5J$n48={tXRt^N^r48t8x z>KHa%8Z^*AdA42Tgm+n}gWByj0a3>NhuNxllt*5HAB^oOH4Hg@GreIb=fEzUHCvUA zzjN%k(mu;d$gFbJc=C@+26-$`tWk3yMNs^Dkgvj7VbZZon1juZSl#9 zB$XKVuEb?#xCP_ibCF3sl_A+UBuDJyU7DN>(zB-)^E5d7D%d&l$P;eF=>(ZME|`!y z&qpylpp%4D-J(sW%)FQb=NXeq z-8Q=%e50{>%;-;{qV1a1&Nvo1rM}-p=sxB*J|J|t>*k+lG(+AYqIR75Lc&tofrRk~ z_VSrT{Rf`F7pdz!i{737qinL(WNo`X*3V3@7FhdCKko*g=V}BSarsU`dp~?W`lJ^7 z#dQiDDxQN@Z*cT}Hs;xjHJ3C}xB2%NKfFZHt!q{_>PV^V9^Us{Su7QJqhw2UPX zc3N$lx`n@<+osqIR=}RsWSeR4!+@iBUXCA2w|qYP3-EJ;*bJRrE87J6D2!(qF z)*^Jp9NCCJy_@j~JwC$lR|pV=ICYciIJ?;p{ECQvkqTdx=Jpp~Z~qqh>MOH0{iT;& z#-Nct=}qS+Bgu?mM$e3vb6kOEBY<8~7UNZmS z#VwLM{oZr1h>TAY=!w>sjw&4{(dxa4U=ni7_w$Ca-~*~wE1pV1o+R>5%fi1}{`l1P z`ILg*;cU2nxP1D{Wd{}~2Iyfa0sEc%8kDJ=T2e$V6p`bFep`PX^c+9y@8d+-{H)LN zegDnBf1ck=SM?_;UVfe%nAdR z-$11!wC;EN)NTziB@mQ80{gk*f<$?~qhy`p@&ukSo~FhmV@r7liQoaqsTRimJ|$I; z8|YPHe@V^Rm0rC`kGpnb;rR9Jb<};|P4!+3w@&?Lx)){WhT6Gv_edp>q_`@z8%gfd zK0j15%$`5C9DVp%(5sWo8zC-5rN{3Ndqho@tw>vn{spG6@EzT7{CfLHtO*WF+i>g9 z(G6V}Ba(O(`OH3Wn9V+`h*i?V3b5vn5~+$n*t%U!KAsh%{DP$&;THb884+qgx7 zy&s3;wdQ-d1=DTWdE-YRz#oXyGXP=AJveOxM3$G=H%>{S)URS3m!L`pY8ccsE^* zj{2Guo)_(5CvOad7tw&2267Z29L12J^u|chA#T|GHMB6CBT zo)6!gm#!bP_8x#@K>_eLXU@2hL$3&5?9!?QbmgZFlZ5F0o#OBO<1a7M9nl8x8x;=MC2N;*9b+OFw0vJDm&fea@15%e2WPS z2V12o_!T{NfQX`n)+1Fy+rh5#Kw*n&+Lopvjo zuquw(6#2HvZOT^`pj@@JO80RA#9Y{%z6!vv0S2peQwc$4u39M*&1YWs@r_cWEy82i zQiIsb;j>2EMFnzZ^V`&ao&kuNRT4QXAlhi7(O8IIip9Mfi|RWGsDRlG0&#wo-q?Y6 zRofrLuduk)dilB)Nz@=|QwSGs=S>VyGW|yhM0D`FZfKe#ptkSp)q#Fw3pTPSFZs&= zZPUsyW_6*IQAiRa@*@xl4S48Fmz))E6DhhgT=jS(QZFFxSwgJoA^EcV3*f8`Ms47V zTR0}2+|r%h2KQQ_hZDA-8Rv-dI!JQ{euWhv%L z{1q{3%nHU2nK$CzpJS#KWTZ7#y8JG?=oG9&@#=sdlDhctL~Jq5KP=U{y_2u!0QDC zm~R(|dwDI$=?M5yd7rrrf{@+gf&IB(dViB8%4!HQ?DOd?@n83LSPvX;+ZeR{oFeW;fP7qdx|NZQS0l-x8CKMGv43Q!DpA(_%*|n0eA=)M|}Co zywEhOP1^yew~E6Sc(zfB(oYB8{A`VExyon103+r91}WB+NFq zPF%}UNtiCP8-36|&gmfBTM2c0Fs%3$vD|{_r+GMBr`n)MrQ)y|ziS+S;9D^yj|s_t zz3weB8C0%q%CPY?En*IO*lYWI@=@f^e`IwoIf&yg#&X7w9Y}vCgKpy&HmB(pfPUV9 zR}CD5n+@Gg=BZOq$x(qsW_By#?Utm+U&-{z?w(s@-pNd;h1{jP9P;Yn${jY6#0 zpTJk9khZ}Q>HK5Q_wO~G!*yp1BqnJqzLq0a6H;?_eCE|o8}(ZzI;y*Gw09=+F_X?n zbn__sv^TE*lXyX};QLhYg-~1nJ4>^2oa6|EAgW{b$NkjOv-JO!>;Xn6f_sYwAgOZV z4XnfW2|B4R^qVn0H?ep9MS$hD+KJpTkGnD4Bc;iH2lE`L9sPxqkEqgFo)*U+q2*d> zMMzqrvP;1J0kG+ld%p%#Ias9rolgQ-j6$LnwhR8vLU!=#?n8QqboqNskd?dcXNyLY z;$GSINw1QglvEnM!#GbO0FtesNq(0l8NFAzSq6DvAFWG~kcF6&0g8Y>nh%h*1=FZ! zE5g46NGfg@Tx&vKc76|OAUTJ{t)88#fPjon<7`2};)AN%CVKNb<2(~2)Qt4!{s7ak^06@ zl2TljIINxu+eE|+2n6|-B;d(*yzta>Us$ytC_8>YFn%qIvur0d*KdGTD59b)`6v;O ztoGzI*m4l4!m?O&Qh%_8Z}~UfMT=$3ui|Go5kEN;TQ$qqinu_z;dLP87dz*(fiDeI z{o$1aLQvaoD%;-ovn%b0-{+z(iXiY(!k9%)0`ap?M#Ajb(N`I<8VNc$RcP6>%bUtW zV(;jbTh5t&c9C>G$wM}KjVT4>21mQ=)$BZbInKu5*^;c{VQ>1ug}Mjw4p(sg$(pe(&b(UT5dhEQjXIa30)}%-#&i(Stb?i zJQap3-Q&_H!iyc0i(s}ywd=-Hwx5_hwqa{Aa%5vXCJ0+Nx5jHDhY{QNB{ z$X;O#l!60PhXB5eg-<9Uy_?XfLb*BkgZ0JD*8U1Xgk(OM=+bdbDx#lS}(n#x!VKMhY zN3^wRXi^2+2RbPMeyLuxnNznh2*o?F+uGQ&SWLPWwRwj;Ft0OVZ<4uJoJ9tI(zJpo+*_41l!g@a1a zwEDU!{^Ik%E4g=^i)@cNiKP&GctkwB*V#xOxI6Th+_%>Jh0fI!U!hb(9{rFt^duJo z@zT~Kqiq+_4CW(+-ppRPyUgQ&I#?p!hs9Y&r^7R01V<4EQc2C8&u6 zz`Gg-GxWI|RB^qhwSo!HZ@BCU6iPSE$WeZgiWh{fWw5*hUzRsN|KIwl#Ot^u~f& z4%_rv*XXHsP8TUFvW4Wwm{pO}}|urZ11Jk3YH^rAOu(Sqv0l{h*H&;a4_YTbh!+t=B5D7tt7G$zLGp zCumbHDvpjEV9{{Rtid8R8x6P0+s>3w*?6{m=hsYr{BI6AX z6L|+yPWL;%l3Gqeo-64k2YSlq->2HNnvH1PkLsAZCnPf@n`0_kS;%(tlx`^mo#(|T z_L1IRHRK?!+W|-hPYzQ>k(7Dmcko|%n1PL?z5?1QY&T8MX$+Dp830s>)|rF<9K0+2 znK6rZ6t~wT1SuBP+ivYT#EaM_EJ@{#t5VAH_xobLIUX3?XVOU&B5reW>u99d;^MG* zsEl_6o5^X?Fl8>~_eMKY<70JfJnLgfO52tBc5m^NGS3AW9TD2g$ocs16gX~djC_3+ zM^*_#WQ742Y}R5k^nGnUSSTRg$!GK-C9gEHLHqkjk@4qQhT4))foRgK{|S5SoKtLT z!p34U_%*wB&bN&W0{wxSvKbmaci*I8ox?nG6(##=iBrwff#)mvmoz1^e9-l^+_CM2 z9J0{NE;P*FtU6NKQk^+4SF@;GV>&MFm4otOGx|I94EAt zY%}x~{Z;8)Kk`~46<@4I`8<#FOg;nruoXCLV$7oYp3Nlb&i2@GypJdd%t!MuAIrPM zudZuc-Ln5)B9vWpe8@KgqO*QstA{~VI=ipBKAw+tP3*z+*jMq0ajcyrsJJSu%_{8O zd$!#P$0i%FjhqTzc5v{))5VLiU8%%G&}sCL!rb+MRdFsOko$!aL;(SK9qtGE9Ez8) zFvkkbGg#FlTUNz7kTv};#@%xYs8T$^Y$9sUrIm%M)InN)Z!w2`oL$M)1T2ytM#xpX zG#IE--J^{9Erp6UOMJR~7gVB1CdWtDN3f~GQ(S1JLt(N$CBh107cC0jjQQyg=}_d^ z!>tLo9SYZEjKnvEVp(@NBV)OlLmjb4ER$A4G!1r82h*Baf!1h9IMFv2#$OtqD~A5$ z(*PI)?@eSjr5Nw+<2ue#(M9K4tm@)dpvzT_zMbnBMe zoxms+h@4{la!cQ|FzJe4)iP<)EEb0ao|GNun+SDLAxV?#-w`Vi*!OVI6-OUkKV6|D z4kaJo)|9^6wE`=9d$Dpn5MIr_9&;zxxe>whshVIDHB=$T%~2f zFYpZi6V^x7bCX3B%@$P0wIe#>XY4^p_t@URg)s{_%;zl#yavQ625opRnsB9-2-X_Q zeuQJ_o<^&5BpI#*(tres!&u`R@8S!G3-by^J+m&Nflf|j_+Dc?CZl~BS)tUFz+j9k zIPqHblQFRgZ7xt}bQpn?iZLIiusZ?FkZ%u@B-Ch@;YX~Fys)-*9u+068JGxwg2mpR zqsql+ux~185!TgtQodr2*2@RyDyZ3p92efbd8aNU^{4o=A`Jx(*#*}GAqZurI5quv-u-h>NzA`i z+B<&i^&Nt(UhV#!+nUp-0`Sds-Rz10wG4C&c}(*#w?`yu`-^Q@5rxRTuthGLBn2F{ zz*lHP9O$xj?dSN^$pfz%#XK7}XfRjPJ{y)`3W|BPvjUdH?*O*FcIp9mulSLNR35 z>jNopq~O;EI~o@H;iPb80+3*cAX%ra#oZorG0H&aPvE=nH|oOoU<#Dh{C9Vb2Yh=k z&;>G6Ye5x_)Vx4Vul5m8mxD?l{Q+d$QRc6yJx^9~yC0B8{}@M2@b#ub9YkES|0p_y zxplTcN3gkJ70`KOb2m94o#O1;MUwLGHb42J2O`yeQTP;@QX%&eE_i)61hAXtyG+&| zU}{Sx5+X_#KK9~AJar7tBH-@4HuI=iF3H+-75N@cbd$m>xAO(OtfK6US-w%XG>$p? z+&Y_Is_OuzR|~}Bfr4xx;ro520*e-ajs5=eF65)+0#0W)VP1~v1ueZh=N8p$?`Tm| zb*O_FGa5ZK61u(C5-utLXAl>hZ`TEugg@HH#1DL`;;A(s(i&cL`9b7K@P>u@yKLoS zS&1M#6I)#B7xB$J!~+UyT6F^OZhX9B1|S9>-$jleTDs46b%P^$5|_ppLi-t+h`jQ6 zvZcMvpaNv8E5td;&A|N?t4Eea39a+zZ%G~?QRz6oZTzH!L7dcoVh#b7{`V4l%v&Qt zh*%$e5OO(h47tFucGz-LlVXJC&a`^>@q`iS(RUlweqAP8rmU+^ptS_z=D2?$&uQfz#j#W^#o$`DEegUEedOWvBj{D(<-DhF*&Ya5G64#u%=r0oW8BY%nKve(Eqypsbj5Tu`vB)ut0VqpME# zA;n|XB_uwz($;AHOWD;7&F9UqJq@?i;2w#>rZ@W_?@%KjYX4Eu{M1CezUL@XemG4$In73Mh9hJiN?=0O+x(*vq zC`F3i{=FqXSX-5 zp+I!_1J87n~SC8TD@Z3=WKZ8ZHW8ob=CJoMF{ z?BKEeGZ_ApcXw;qo=OrPe^8TZP7G1fgP}_!Rnuwu@xelC_KCV5o2(@Li^3(l0E@W- z7)mX>8felt$I4SwT%v1d{7K~LUT*HcB-HOq*y*ne9X1E|KW%MC{N5%QrSlry*{%yh zz@%irW1>%Tp^@LO1XX|n?b6Lp$1^lFolR+*8I<9Q1HUV2FsyLZC zNhF<=g28`gCvR-wR;^b}KBi8|2Anz|uQ()dQjD@ny)YuLp$ej0TjdY#cq^z_#vc=| z6#%CK)4w}4pfA&n*?xfUURcuLw5D!A`{`#H383{8U*sHTs=(MD2ux~BVv2Kwg^>V=i=FkW8Ono#karz+0JnQFHxxyp_IvQjd1F7I5#7Eb`2LAiN}Z1MBZ3(I-k z^gSjln&5;fE$b!|i!>dRuu__Ia)wZwiG5zXJn>Z_5blAXDhg3RF&iZ2z&#Za4NT+S zNJ*|+03y#V>oL7LB}@{_csexSlf6_oUQBW%jPP4EdK;v5fUlE}An z1^T!5#1WOUT}js8g4Y>;K_T%j9YZq=i4PHH7$SZiz2`lCJ9wtqm^~R_b^C2Fr8bz7 z_GUXp65M7mxmk5OZdHH}+lDjzw9ffmwhmV#^~z#pMkRpGig96(Ij56&VIBm>SJ{g?R;?Y!A+SnEyvRqm!1&A} zFLREkoZ8qJM7zF*x@^IhZpuODUBml=*~7eNe&@K-UaGci@mJ`@}u3ft#l#C0iB~Ff8Iz0=S|y z*&PaR8GJLWuO%O_?WaMScd}2F4J$SD^tVf+2qp%p4kA;-r?a38BcSO}=u^XSObORM za1**73vLy77FmWUX7fV)g1vXqu9jDDFsE?!RBoHP!FoA^C9xeCmAOw%xNx7X4h$`@ zK3V6O9Sgu-F^MY5gKuc=`G2ydMQ_uC4K&rdR3n^vW+A5q6ol|bB$Xw{9LS&aQsm}) z+{!Yt~0axP~&xDl^)}j!?qPi4mE%239MRR)DvZOTU`>sPQF@lJ)v7m9j2OIJ`7I`=V z)e~;&+^!Ag$mxvKn~zrmtRH-fAg|kkI5US5@!SBpcpZ1fXu93lGHwvBUz+Vuz42bM ze0uCaDDZ8ijg(-vO`WI0oyT)PY_kiB#zYV6p@yHRFw7Acs|2#rA9%mnJf>E_5K#OY zb|tu+EmkzRg)K~za9eqoV=)#V*%2f^V3EhFQ4lUBU?4Szb5qX4qdwy?9$bX}@J119 zF7d63369;ThD=%^1lu1gIgLyP#aFI*l z*;qmWA0MFtV3aN7@$W z8+TYKlZ-@Np^%%L2+X%wky~WF$)^nER#0E~z>&_QoCKGpM95rlXl7jjgMG**tOsX1 ziX{pP^2s+;VDNxvBgyKb$@sJp;%g*DYaR)O{X{{NAW|n226b7BQ>5eAmbYpugY_Nc2-XKJet%xttWLHchlxBDNMVN*xS0eu!Z|1xoYl%ppXiI znC)A_`?1w*mfu^mX^;E6YK{x*5UP1ovR4jP@ei;gC=TqBBtCS$Yg5VShGlhK+wqF?xsoGoROTkx!Gtmucw_M+rd8F5y#v90qH5@{IzW(fX&4QQ;?e$a>ZL zJ{?Z{t(0ueunENo2D+kOrh&R_y+9sm=UH>aPcHW?A~cP5YVad`WUI2KbAyU1!M>q4#3Rx<}n4%)Cnv`E-Mh^W7_q+S~Ww3UEhlL+ph$MyT)ilQ0)%(vF z(?`UjG!g`5R zZYbECIf|i9!}+MPBSki#KW@;SK=Ss=W;b+4{_tL{Kx@hmTTFFZY?)n~(!y?N`PUb) zd5en}dY^>gST6Ai&~?-(**J`kb!Vj+sXvwTY~n@Bb}_#{oxRX7rT2M=nSIH)rTQaX z&FyDt0*PTz!*6>G7mg+)jX#^@oUSOP(3@64_epcaiiCMaKC2wNA#-k%-ub|0n}+J& z$M3;z+Y&*~oHg$V+f)kt`~L!{E)c7YZ&{7xEHk2}^Mh-byjR9e&iPUsM%|}(Cy~{_ z_$b&jzcu9M+x4L%yrhBb2S1(S&tIk9IB9Kxg_<9(LZWK>LoT z;og+~B$D`H`|?gm%}c`mPZML5;PmlFKcV3M7TW!=1uwI319w)WwteFZATBW;Bc4~! z(C-7Xf%k}*ls~60nC^0;U3>c2I~w|Aw056J)Umh(v+Xh3MTpqV;B7_F+KtCha+JfP zhW6|(w$0}SeLcKzyzSo@`^V;-ivpb5g*zk0WAB zC9&Yh)O+s7B7b>b*>S>+d@9~`0u-Hmm*8z_(B6%WzH(tYG{v8> z;naP7Lj>V-IPOXV7Jh+Dv|I+9??JARQOMZMRnL{ZQx4XH%m?&j1e(%OXSt)oh?$%U z2~KZbXPwYu+>+8t%lFSHZ6?ydWIU^J7>`t?Up0vv$4_GR&8-F-U5%Fsh(Y41e)W$) z%RwC{Tf669mSx)EqXFeA^liwqGf_xTtN$1o>6^I_H<{iOCLm^K0mMj#g=MIwEa$?_ zw1k!%{dPjIG_}AR(DK{gEe`4~)A8~np`&55SL;=7`R0Mw)Pbuh<)?fUhIa|JV5!Muh+$rA*~7JIMZB zt|V!H|Gne7;)uEI9Vf*%3XeC>h$2MYz>(vSNQlf3P_)Vz5e7j_H&>Yu$+ zqZg_6pDo2fn^nfzk4|i%I+*YBFnGm-(hLFIU{@D(x8={wX%9&R)Ib7ug zwH5#*{A><}@_5Uo+|Su3+xT`bdTqnv&Dq(ep#x;jg-4CC1plr5p#`KU2>!t4ZRX3z z7vCib_oY*DE`Lw%(pVr*c3mv|)Mqti79T%gdY`&}iNIpW4Z7{-VqdU^`9hAb=@8q* z+M-W$+_hv78B-e;ex#r0$ZOW(ad;l<;|w}Dcn6OQ6Dux#IeyFOCY011vICAMF@N++ zN-xassifav+ix~ZzyTZ+2;bj<(I_ZlDEcNrJ|TJoDSZ;iwm5$fx^=*pciEixdI+S0 zTZBH~yAZkLt(S|#iJg6?e~K>4;>~V}$ydinpCk$6E0KD2m6Q(%Ft~4Y3+Fu4^f5mN zLNz$m*qExUdxg@6x5NR>{W?Sx?bs4&vWeB zV;2*MyO*-tx;w98z)u=;mMx{_!xi}2uc?4TNV%|P;OfK1@r^eSzt+qAs`B!D*v)Re z3cb|N&IMf>h3x2JWLx7f9ZVp6#roi+D}Z6uzK{X?9WO;J?rhDJ^rd;9KlbO3uiEez zTUGW2eLVOt3@0vp0`U+n6c*xdju5x8Y}*zxb6hWt!cQOpAAbi@yELjKSKV+D-Nt>6 z$JpqGw#DY9yBIaX=%ERh$qptYp>@In6pfAwb^Egxfg8tDJw_HQvb85%lWn}O629b! zi4xVdUzATB*57Ln+K{+g2R^DD#~{C?o4eD15ZGCRDty{WzW9KcWatbd`q<*iqVHmTClvvkRN-MoKx9BGuU zzlw+XMma4dxC@gYaPaQCQsyuUypV4qBPFuM+FlL5w`Nk!@U@5P)PeVV!5;?_f+6s= z*ZfY>;qYYN=wG4R*bDFtEyTuIzv-dl*W&(?5Gl>7>LsUn)YIc`_SXNm@g4Hr{)_tY@G9Pi91bN-+NkER(%;!xlK$9jJ{bf0b;`$t? zckjc#BE<5J2&Lg(+$PxF9Nt#agLIYj1X)fX*7!Lu-(1Wo3_qi+%C zZ|MUGFOP4VWw@V@YsO&kIQ5Xv-hcz)BU2FAndq>eIS5Rk@?j&W(V?whg&}C?Y!-C& zWB1Kw4;f5_X1aWqsQF!Rv0zu2wnUrgBpqHUMsCdm@6@u8s;K5YtwO|~Uw|`suEq%> zV7nlfNt`Ro%}c(H4+IW~BnH^sANMULm%1AW8<{z)oPU)~Rn;~AM(la>Z3+DVN8)Di zG&oE*(fD$0reHCM!Qm?~Sx6VzFKH3+Gr~mjKqK(=LQXjtLFFvA_4u^%8&7EPJ30X? znb;^OBw*|I-Baz^39FIU*+Q~CpjisOj+2Gmm7oynpkMSm!9#uqO3yW9BWw6<;HO`} z=XBDfQ{+`Hr1qA%vsO^Qz|PY&q^~B!6Aw5E!xAI4_DsYbcX%h>TjWVtxv(gIqJFed zyzg31!3qelBrD)ktZHW7_yUpB6U1I}Et)FDGUt{myrA>cU*mWjZa|JW|$@c6w>))1PhywSxC^!4u-uj75C}=#;ly+4BpRmI(V3!ntGB;e!y({hIbq|KR& zQ80F$E~(bxhC=G$YFNi12(D36-c2q^poAIu_S;B8f%I*Ft8RO2Zl^?OkZnj+sTGdF z3Ual7H((_Qb_RC?72(l>_;qo(MCaNPZ+*|trX0YpZ!lavYI@D5(naa zRmqj^=}idY>~Z}Ly^}~7G0tN?9Q-TGK%BHKzMxas zuFqLan#f9%r5plVrp0$FK+B4WKUPMvcR0=jhSes01YLoIs)Z4p**F)0UzrjT_+9C* zx^-Ogs3Tk%l12#oQYid)&m`4-$JW!RI-V7o%f*m1bZB8$eBq%FxH_{r`?Y5$Bc&HN z@X^2(ey_ktDoTr5Y@*SS*TA(~6pi0ygCr|Tpb}ID{T|H4+lXvJ>>pZ8v*YF3?8J>Y z6e%DrPmlS)yJtOzffyu&>`(bi72l*bY9nr1D6}BA8&yQ0ekp%;__v)YpPiF~Xa-B& z@<&bmNKY{)$*FD~MZKXt*!O04uMiwOpc{;yZ8@ivJB=nI{C+jB15JQM6Zy!Fwy{}*d- z8P&!cwf*8+pryD4DN>3QcS6w?inTZtcXx+Sw73*&afjkAr3BaFE$+cRXpkf){XfsU z);XWfm-p*rt;}RHbMJlceOr=#?KmaP2d)JXG3r(5Ck6z^ z+#4EB$D)=OnV4k2ew^$ z7(pCdIt3gHZks;)ozAJ=M{!q;2cMs?^8wDsm2TRv2vg%PP7?DF_qSyt=byvYtW81N zQP(+GK)RoCxW_K`@WOYKmn1(i)AlK)n|$|jX2HnAJ5-M8C$dXFf$aS1N+fVG(#nmv zIB?nE9_6+AnIseSh36(5(dmI|o*PU4x@M@+B>jVT6$RbndOxSl72LMFoz>xN^NL@H z=Yc2??ZRVasP+6N&e80>-S}IUHnhRqEHe-sUUn|b?-Y@^pQt)Cfa?6GQ9byq?dkqB zjlac_`cN)o$_u&b4$@VhoSF+OGT~31i|kR+qwS1V&8CDZ;;--a(5F1}-Sf{{U;SMD z0FVK9J#hbeNA70rF?{Y)9OmkL2)}5ZgkW-a<@8osb~b4j3SrLMlWbyk06<8?GF<2z zb-~DI1BOeTG@T=trw8odM{eo5pI-NCt+^qmzZT}-fNa0IeYFz&xUwzoI7LP>$O8TR z`t5OJLIIk)4m?_4$)H261v>+Aj?%d@)6s(x49g6dS#+6W`S-+;PctSbU~(rr)sTW@ z`f?8SzU9!Qf1^b4YP9G@HZ0_c9v9CVBAT62-f!?a-*}hgy zO341bH`xY`a!5CLoE#N8Nsw7(r~KO13+99dCzU!DuxSH`;8|c~kUxRsYigF5FsYAC zAyC9DJEDx}aR^jYHdUgeaU3-TwWA*2FeORY0#Gb;Q@Nc9&H5X?f|w!?M88L}IKDKo5*~oTpEjA$f?u-_WH|bin_*y;P4TNzk&| zzbKuUi^AhWEy%~8gmz%X2t+`;2i(SG|@!@E}yR| zGHs+${KTS~`kN^^`+W_qo+<0*Bg;fNI5)nVPW%sH5kHRPE)ec);ENUIf%D>+J*NT0 zvH#uB)#*D)B)NZXx+GFlgzNI&ue>s(05C7Mmj}5f$+8!SX7e=@WyWct7o2zC`(}qN zr1ODq?+@(NO;ZnYL-L{jD9r3i9QH>OtvK%E6)><-U~1mV4N>NhjrDHGV03;mS2>En z?`g*8kSleX7?B~m+OExFogx>lX?Q-Uso1eu@bp=+Bel&ou7wi7KKv``z@U(8u4l+rpqGh#zf~d2|x+vlOr}l zDb>7)wcu8?%`y@L$#6Ph%gLb9I|1Bf=ti@>hb&em1fy3@RJ*=N@+VQ3ylJ@dO_(^s zZJf2maWYtTJqyQ^JeeVKR>z10GPyzG4WoP`(GxE^3=>tzI(ERqYuJ<%7AxCC5F=AZ zVx2uR@sD=68jGV?3s?|>IlJ{7;q>cDLZYH>>Z*umpih{dWW0+|A&*@JvTnl*7o)IK z@&_U2ZQq2yFS$!kY%vUOGrCNld;35$xX+n9d3`HlLC(c>QtpO_UEZJACHjdmWkwJ$ z3PlESrl5PeQN!P2nH*-qC69h93|~}mI3|`b24o->n4E67g9x&uFVVx`8)eZh;9G_@ zuFd(;I?X~|@A~1MfylJ|Gaf{zFT>RvCX&xRx?!f3mq{b$hmoHK^7zaxS^&QPx$J^q zcNsda2GllE; z2e6-7nlC=|vmR%*cKS65cSUcDsJVT#aLe-)(fKpee5O-vdbh`fDX(K>c;gPq*?WdCCRZq4v2H6626X-cu z3u@jOg5b;g2(sSz!R~x9!KL&nT@Jzm`*LpxIYbW>mOnON&DIUi@vwEZ-flpPNU@UOYS0UfKyHU2n4-g`a)_ONS;;|QI)PV;(6+}Ge|ehm|}b)f45 zxwqJf2+e4nKWfTmoc=M=!Q3)wh!HF<<9<9cPMyM#d!<2K&( zy9frLGxng$mpraWUth9g0nAl!Jv0S5hiJOuL9v=CZn6W%-ZhFWHJSBw&a_&DPF4Og5Fxfqjcyrf9sXC z=b9o8t6g_{tMzry_Xm>8sBJhDboFYJ^+4TMm`;QtYs)N$yQo0kBgrM5IrSI z{fm4}hjtPO{~l{jn#WlsSGdsV1|I;+=TrX`M-CW-!F%Ng- zO%{Ex75RwQSTdih70LL*X^{mgch>Oa7G|BMx0J_;qDqLqOtOld>!?~5u9ArQMbf*5 z%eHHm+R#~Jc5rLp@L7b<51R?pU#g3dhr)eMP_f)qw{>3 z8HoP}#_}2fv#4pb<@XzAXWx#FQw;QlU@6sdcE1;#GJ0ZL^jU%CgPPAFzpY)Rl-eJg zt80>$b9|EJnO)v){pqOa-06{zsbJLa!EP3 z-1;2nC`wF}hGf=jG7m9TTIalu=(aazctqcA7@XG;Hra91ip7Uw{{wa5{SPWr1Od3g z)OYeiBcF#3j{ly&!yNUMh^)=^=!*b8`Xocu!>Xw2Z4ui?>i@I&Y7PPzRy2BRE*!de zfBjfTZ29j+!9K;FZHYXGmF9M-iOvyu1iR+Euw;r8O4#Xn_bhENIc3_hr6hEDna$7H z?CqNW`~SHvh5x-SAc|Af^khd;{lCL{?XSae#sx@P^H9VngGl$>GNM-XRrxBT|3EWQ z%j&hyUvbNcP0mIxmpL=*{2ww-*?e?bdO`M{UEj$^k!P>|2W7K)gF;p*k7-O?Hf`Qi zWPM573lm4;Va!93FeID+X!(0x?x*Md5!hcmcoxegb-xzRp1wOc4QiDYE-HPvM-p&! zjhnaW55Il~M52nCO-JgEXoG6eF<)bq{t#=4yl#pRCHyqeHpjQp^eYnRBAm#yonAl6 zk>8pMlfg1_juDld!SH9J_r`Afp781L51=H?^y-cRm~8#A$q%>o>MP>Vg54}2v-bCNyn-WJn@$E0#Lk=hWe1W0{6*6SJW2#}CU>V3c)J|>la+i_0 zpg}z0y_-&wXy-6^OS~I(;}s#pLKV`jMEuPCRe$Yzh0s8`zkjnzlHfJWM6g7HzHS7U z`vU%JTIrnK?yM(KeyzJq;!&X4{wroijQi7*vBegmSdQtJS$YJx%p+<>R3|{^eKyyG zq)kOG_ffW%zhD0WeG-smq#);%i}+-iM=Wd%-M1+Gfdw{b)G0hxKf#w^=#>lw;)rtx za$c9gA=XQ0krcR&SdMfVE?{&QbaQyT;H}d zWA5P-2v~dFYUx%Stv3FsUm`a7jHE3#qqP=Vi zfz{T-Em;)zDI)Zmc^+@MS}ZKA7DCV*7mG*%6F8P!Q#o5j3cd+wtJ z3T82m8B?hk-v7|ha^-|&quogK)C+iTxESpc?|9G{!yo^ut8W^S2KpV}XF+?$ofCf( zD#f10j={PMdKmH@II4eCO%YiOez-Dx#CHn5xZlnINPLtE(-LRyy3UfGA}x{gDK&bT_D6D(IXM(ei+KyGomP zI*d==n~>_40`B!del!ak2AyMYKWR`UxR9r)0iAQFGlsLUh0P+|1bi(f=~zQPQ{ZG} zl5JN&kou#*PQgOH(bji}lZ?sDpK#vIkBehv;JY4^Yz99VVpjFB?G;srOGk!eyeCzk z)CK18ay)YBX0{x70#sx;`!t07nuh{RN+TX;^FjvtClXE826Te}!5fI#jB|?0ITkL% z$4#MOX_adcf3h9zJxwK#L6jKwH!rI^p#jGJ&#{2)s2vod=^j!vj%?SyNCe*P9Hcq8 zY}_i>!6)xpqX?UKkJJ)?AGYKmTX-NK~?w#9`)5r2(v8vY()eo~sbvHIB`A z&ztkyExmteCR>KQYg*QZii{cvu0e&kH-2V%cGRzFCTTlwEaVFUY)Pz&&Ias1ib_#V zFQZy`-mrNRd~E7B0S9A&gpZ{iQ~P;5t_h7M(8lWj?=zeNYmqpEd`SOE@Z-heEFmvA zvh_DlUN8*qlFJaZy*W$7(%o#o2Z_kwA91T47FQG?P}m{#gMx0 zxln+!@z#LW<7-JBa@(QOys>+asJ*=ZoM@NGNjl^hEvtUHvJn4S2x~G-K~w{w_^uNt zlKUAM7xmo9n*dq*RALXmJ>7rky$r}BJs>IeFUZB_DMUL7xSRILn^D9E^AGX)CqpMknCuC%Q8&PPOa3o8N_la3cTSTWUgn5U2BavOz`P$o2h{?C!d z-zh|SBRW>az?I}`AmQDRLl4a63F3A#ILIHGHV;i-ESVKp4>-IkQ9{*TgBkgR1q!+o z5~EeQPX!SbYC$G-OQ29x$o>~{x=bM?*VtaW{Pwd&BvN12S)=0%tIVjL@hHjrBhK!S zC<6~<3z>bCSl|D-AC#d62y}&C?Mj>t%sUM}R&qa=w$Pw1Q$N15fMu8%TmF|2uQo$? zQ3k*;e3@DXJE44d$RLS zf~9e})~45-diUSEP}phuhFP>Co+>iHka7?=cN{?r@^@NX17_;~d;kFs^(%b^JsuLR zpNr=puw7C+&o6}H+Q$|+?&c1Xl^Ezuer@Uv_Tkb6XF_Xx#?-c8f`{r4HXHxQxh7Rm zAHsF)HqQfsU<-Vg)O=T{1r3+%Tr<|Z6fN;(F#~Gai$CI<=7a5S9u1WI3^u9via&4q z(SqUJ*v;bte(RFE7I^zB0(Mu1nm+*zLa94}-=TT5tUrei#GmsmWm%Mg zhfyZ?z9^Sh;A_|{C3%G+cW1n;6pgzH z_5hv5i#0tz=wA`FS}=6(;DEi$-G2+pJG9KU!bL4TEqk-o==4F+zeXXd)7|BFyJul8bTz2h(XPJfxX-VoH5O2yZFKdQ2!=ym*7!p9dvj# z97b{xZ$MB{GMW!9?ppVI+=IiB7x7N~J*Gijr!+S_||HlE( zc&BHlu^Xg-fPM7S#ElKn4YB-gcX&^pklNvwXQ_hTIKR*(;0n&TkuCUdeMZ!uyeHWfZ3Cl3B!Qac_z3yr9eeI-e~I z%7=f$%?u8c@GL+NcFsiGTV^!S?n&pNf06ju6d{Q9IBTj~W870VGY^$Zp3GSr%sUPs zi_Jg@6oSKb@1ZJJE7m=7RO2I0LiVRG(a^s)Z?m|6$aAXW?1K$6DC7{Ws8N zy#L=on<1n{91urj{bp1|`J9%OE*6ODW2l=1Uv17_)51!GU!(FvF@*H^|JJ%xoquK*Z@CYqH2LFA3BLK9{NwHAZ@4{)Z^5*8HBq-1Tg{ddv`MHx3y-E zY`NWS74DGx>RB*iYySRu&+S^k<3+&~!DjFv@h>@Pwrn@rmORIrVd)3SFPPT^$ zEIr*f_yHJekw#7EDFAhU=yoGw>TQSDwl+bK-$sDD`_<4r0VKqSH#hih8JK8H0sNjvE z_26P+zdJfJ0scZvNKpB}xMhPdFoi9pkV^2)>a*)fc2q}t<@gSH$%kmt{NOWgQ8=Bn z^>VoWWWv9{FA?@b-9nk*o$D8qhvz7(6t&Y9S5CQsnfxLa{g=D+9}L#pD&Iv9@Nq*T zU*8dxW5Y6^K!0BJMYU-3;`ps-M)D#Ux4|1x%(Wm_h$jFce~x^tog6Np;DN|WCmh|9 z;~ieOZ8ZbvP5?eE7Dc8^%3WgeBe4S{eR~{gr=tZE?!zzC4lI(HkxR(3XP|_DHADCY z|KI_j7JlFf-@TsgBazPMqX$cc2LybEPI74qtlG26ta$E*Jcd3}{hy$zajB1a?jerQ z>x1bE!}i|g1#BjkRdyo}4^X*b;Cb*kuhIJcUomVmsedNQvob!UIN{mhV;|iF!b2p* zCGj#oMUQbS$8EG}_vmi@*8iABAr7UO5`FyyaDB;6bo+Sr4S5OxEpbHS;u`;6h`#l@ zyAx1a^s}_X2nAg$X5H~S$T4pxkh~NV`x5_ESZ<1NM)J>-0phnW_C};@CXSc&@A# z=2QfPw#J*kSBZ1;L**G9pzr@1MPH{_JWXV#!pz-E3_u`@ht|FR)CE9wuZ(dZ@c{N*FLK6_t7A_i*px!4_IiVkO?vz+}6$f`=Vh@%hC=fN zg96tMCvWcdzSe+FfbJ^l8p^}Q$iUn=6i4VeYe#is_5cydcTp0p0u;kfIdqNij+QkL z^F^>cj(`6w@m0Njh=-#=+&EDUvn5VhvTYNPM5phmU0G(!TL{oha$MqT?V4NBHp+*c zHG6G1uYL=u6hT#}K~IFVbnmPJu476E`Dzr%+!$Ujo3aVl+CXF(OhZy3Mb78ukIp*dRW$?K^WJm2ONwK-e?i&|f)d&_ zSKS|9ntI+t=kjiV_Qpdd7R{GuUvrQ^6z@{KxpqrrIu~^CceWAaOZ^ipqP`BgkDk2+ z!(T{Oy&>434S%dI5iPWD^uPUQpPh&i5_!v~?ki(a94EjIv(9@)QBnomS{z-?0d>Y0 zE+lQ^)_{XhdeU0Z?aaYv+%vJD+X{B5;yn3dTBo7jyZui!?TEP&$e&A;{_k-hPusnh(E}136E-FPFN8(8-xYVO$>cx7+$#s5B z7uYcSQwM&dPpE!J`{$!g`0!ITGoWz~%R8xa4HlOzRZ@gP{Ly@E9z)x0*DMki@TfR) z{@kT$;r@_D=b~=RGVrbCzoAq(O>hCKm|oIyyyuhuA5x*2ye^-?|7%!PE*8lU0HVdk zW#k;}`jM73qi%RpX1+YqxapP(;B@vH(I>qWsEm4+FWLU(gwi>v5)K2%=#NoPVq4ca zzp(OofJ&7<(D(kGQ|ijbygSTv@Ox`DNPks>v}nq0gKLQTrqDye+-@DgqLm9mF=vN9 zbs%JUE*H1>H-PXf-Led0KTHdhCe|hR{OSN{bEdih7q1bY^smgl`J3+Oa9KEzLN8(5 z@RS!l@@a|;hS@m#=Tn?rOaU36$ai>S;l!T%Bm#k@djeCcv%fuW!!2#|KU2@lewNu+ zJ6=ebA6DVwZ7At)d9l27y*6to1U@^{S`9!VZ*wUZ(;W|trB;{Jay$;T{)+9Hv^uXw z+`IOr1_qz;Xqx1mt^>*pk^L?#_`TwTL^|M>o7QIVUL-tddh68a!8D) zV~kE1?(tdsJm&n6C}aA=yZUlkH})sVnv?m@fccLDgpE{M@XHYnPltR0b{3kR)%63G zshCSl32?>pdiEg2k#Sv7mGI43*8dd#sok`$N#3(nYT{ABIfg0a(l18zq0 z&LVkODj0NXbszG?s;RlF<~NNX>&~5=%EGtOO|R%h-QPpInw%~>nq1fNebNHlW_~Ig zFW;@hABviRp8E&0;Oa-#fr<@&|*;o6x^QqF*-bj|8 z@Z57rF@BLxW0fM*iHDJwTJ}P{F}mlIos1#@(FET+e-orS;L<(q=YQb4LHPp2q^pCWtXU4p;o*t|vEy31q7oin~;sNsR9 zm0JOl{Eq}JOKqID44umm@+cV5$mf#}21@l+iuhxrkyw2t_V^`VyY6a)k;hrhd+UD% zI$iGI8v0S6vZqNYGdPuIcqrUpQ@u8G+nGXTfaGmbwdj|>B6^K5O|5M^58J50^(m24 zN-)fD20MQGUOVYAgg(ZV)UV?5H{o(d?;b!`+jEwv{G%E!*INLt35@V1Dmlob_zN=^+<$mzU_@=z%xJ1dn^)6S6GFL1nckF` zOPVEcO4>omevg*dp{KpfQ4PhgGF&SP$bE-P0#Ko>G5%Jw)L@?cr7si<6II!q z-&nc2wW+X6+QL5#h%ItB$rC!-=%}UrCZ*Wmj0^o>)LXNa?U*WDh}|Oas!Z7$pU{A6Xm2vD*lW^2svlDn;38OX6)mpZp%JNYU@bFpALBh*4S4 zKDOiv#*_8%Qm$R2W0>B9(||cOh`hz)5d-(aziiOpRq~HgIT|!EH6a#f>sswe^uZD%sJVoUzMF~?sx^40W({eV= zqf1OGM_4Q%rS*X$p@QXQ(x2r#W4E|$!%9jg_)1R=>!@x|8`yqyRMEDm>GCNRZzM&f zg18%IW;B9R{hrrKHa2`9em%ZN;3xk)o3-(+vY2Hb=`f%#TJglTC zZ0y@IyJ+_s6ccf|R*^s?*I}4cV+rD|3mbXjpGpg^Acdqo-Kpq?k>g@~m!|m^D2M$t z*1ZSUl}Q5B!gEPXzkNEXG}DbH;P|4OAwymceE))K({mzj(wkAyjzM*y;e zZW))QpGXaVS)0^+En%lAYF3ANShir-VOso${qF`x@>K;s?#Vln%iJ<55&M)dx5<4S zt%A!2^zzY&PNkdSn+5Vv*#)d5EX&!J?*Isq6#b3Kcl0n-@-(5NFbW{6MT+ypV9D!( zhOJvoS>Z#PuW@uI$OUPu$|L`0l;J#^I=Kg5Ok|(tkJPhnf<@=Odkl;^#o0Nv>lhn& ziypsrUPL1yIdSJ!rgg6rm(wg1J>-l;GVo~_xSTUzc@h5lZL_m%_2pcQu|~WE-Vne9 z*x8GeoxkcK^#Am1YWGVR#toO#0J+eU&gYoY=3dd)Kiyw-X7?o61VfkJ4 zG(ECym5EY>acpU)SB2K_VSWkx@eV$+5FIPF)l*z7+prAHh|svAOBv6E z><{ez-T1SpPiR2+s))=-k+uR21+nOIzb=$#8sd5WwR)6{kCs5HTN$0oy>p8+b2vVS zQ>@@VZF(f!iBT)@hYIC{3S8WAyz)0e5q(c(Po?1q%Qoi~VNP~Qz6^$TnYc$$RhMo_ z!aiGjBs7Aw$<>@+RX40=<0_o9f!uyqTxF+5u5g&wL!oQgDNDu@%Js?bO1@CpYj?Movs{@R+EyF5pDTFKXQWgO=pA`B z-f$Ef(?ykt7?b$xZk~Q;(*0^K)lK|bUB!gdv`Q;yZ4^uoDw{;)P23GthQj^v8qXH@ zX1}Tfhh*k;U3jLrV*Gu1k-HYW7%M}0JN^vjRRV-zQ^Fxv1J z`>4N-0fD~=zN%rzpS|B2pOOr^ww8$qeX7cq2B2%RkNES#=I1!Ern}05eq$3L(l*Zz zQ+{LMjLCQ#kTW0Rke?NXk%nm&|Le!K`++XGO*ASSSU5~lO;*O2AgF{QKRkdNy>0A? z2{8Pf9@ah5-i*E&FKXK@=~rGDXx#JuljZQ($T(9O?YqDKrKJj(b>9>eH2#C!E@bk- zPN-J5Y+?RV!ISetym!{S5=d<*F9i91AF%7A0$Ak_g!EkfhzFfJ5Ujwbo{JY*E<~SX z%^k(1_$`eHSs@FWfz@y)ILR(ki>wE4r9)Y&DZ&&G!|f{+49astWO(B3k}>U8Dr+Tw z;J49Zd}CSgO7i#KXTlF?o&)~U&pO@@V%i-N2yB%*b9&@ekY0z7oBLSyDADiSp>jKe zeLlrLrC<3QhaWQ8XEB@xvVaR`{%=gkFL&$(Na@>4Jm>0UgqPK6e^jDrQ9r{UQ4;WD z9@xVcGu&yHi<-TRN-RsB<^#qb{?*ZkTgnScdtgW5-I+PbhL4)9|Dft-z)#B??UU2@ zD&z4gbbgE568ES5+3g|tTUjddtys&U>{#uW^6cwo>DQbn7+##Hnh>C$8ViHSYw^NC zbeghY2I);Ez?JPrT~<{1*>psAxqBRGCsd^ltkVp!^sOcdhS`W&`f0(+8F0dA*3^{b zuj8d=IUqRUmfFB8G%KrW((=lVns<0N#hII4&%BS?7}`ce7`{}z_j4?`Zc&qM;Pyn< z&$6Lw{~$BvqV=TfKyGRHp*5W1rEk6iH9jSr4W@@%Quylc)usa8CrSg&0m&y? zJcynmgXo{+$G5|=v9FCC5g!LA!#xM~8AG)q1PFO7M4o5}#!DieEYD!X`Idr%t$CgtigDIc0kCG5XeLLWb97sjNfOiT@Qyo(yB z93pej&1E2vdRVFV++Ua$bT?nwD}pumJDzL!)g6t+@C0JK&+@8n>+|B9pk*n0{7 z!&)pk0eadH_fbDRN;ST{K<1fc(eEh;8Pyx&m%A@pFDiJq5voKAvjYj(=3vsW_>VLO zTIiJ5)B7P2Wf+_p-^qN5e!PVdcB(xU78)inc;@Q=UV=d~BL2$_CWJ6>7heYcu4M}M zvtNBaMw?ba?wt;vXDy@CEK#N6wDTxg2d{gU>T?>#d=p!sYe%~Zw&`iAS<f0g_y7 z`t*?8!K}_vfVoW(EeR{Y)KiCbPcJE~?oDc9V3L&U79*B}G0yf7&-8*xv%9`QyJx|} zy0my_Z_<&BWKCSdtLbDz840;f+4PMrLz7>AryFkEIyc297XXU!Jl=IB(w5gn26h?C z1Z8-dx~BM4ttN}xP)hfar$S@X!79q8g)SA;(uc{rlazNI2KVXq!({la7q@N0MPqJ? z(WUPs5XLr_>i7H(H7lyQM+HpoYIBsVB(DA!3N^4pRgU(&3*7Zy{PHT{(%f!_h;awwsVJMl zHix?1C{wxU%nQ5$hPxUyHyljBLY9xKEZ*-Y8I&b}zY(%meJLH%qN)I+d@R6@s3tY@ z-glcuCpu1Ty|w8e&3nerxD?!YuZ}-mDRl2d1X~vt!Kb2<**pIQ1l zHZ$PB`}Xn*@oR^~IgOlD`PZ;Ej$9q*%ruYv%L%HX%`VX1X_GEtcw3SoZxJ^_J4a^E1WktvIT{1-+(`@SmYGyDOi0PKL)C3PJ*d;$#Vhe;riVkQA@3 z2MQB$XJ!Z+`*x9p-$E3^Uj%<{aYTh7kOR^Xxz`MbkC@I*ISj4S1|z(a^ak1ULJX;`I~6L_)m2NemtsWnB1mV^Tugn6J9^%y;%g@8Jd+dp$omIDuoGwCh2Lhr z$yS0dCLzHPJaeYs5Roy2A*q^ZLuRbvDltx3or>O#=P#6ZGh`9|p~^e*IO#d@S3)kJ zt#X27LOvcFop$cVTXjMa+`2_Z97QVvuHfb$G~`-~6zUU9>IF#-c?k8&nxssTf?5Roz6<9qCHVO8J%c`A|qd6vfwPos)r_{G-d6J(w9u9t`P7Jz8 zGbA3HJ;|t2m7>XzsWaUYezW8h7MavLR5ec*F%bwf4r^*d-;k%IUO&jzVyZ=QmVLIU#}l%IbU32xb(!S#TWLZ;|1 zo#oVq(BO8uV<&FPaX4>I6Kodc9V4S%W<)6_KE|Nt)KbQSD3qU==BKh<-;WI(>6CEO z@_b<3^fbhN?kL;jwxhhQHO*<>W*zfsAin@f4H{C9E1ibE%Om&XmmqP*q|f2Ktw`RM z#O+acKvIWMaEQQtm{k%i66kbgE_t*tj%vevutTnbcG@raeT0YsG zoUas-B%cRrxKnrZ()+2pkNc(ypK0Frz7`4;5!_W27F}Xv@D6eS`9EeoT;Hp@7*qkG zDNX7)xo3`Wsrr9cEQj{VPKNN(VWn}#3bs~_V$^c;vE{73bO-&GN0VE+(A7Jn6%ciQ zabGyBpUFD!`hzh-tU1g4QJF(MZ521hsyH8m=@oL2kg*w?vIa-{&+{#EWx+|W@*!-X zD2KHke#FMe?h<3D z*QJicZ=kZ2fr14&Iwl?Qy{(XBsr#l~rq-ZX3gvWOgCsWj`4c6qE?bQkNx%zE_?#3H zlTNP(AEbFGHsoG_f2xE@t7TV5n#dPd&fNFyUkTv}R$`X-WAkNg1*pougcruVI1@qW z*lNU+Luy~F!amIniS^ID5{*9{&n~K|Tzi4zDRVO@ zCII|A%OGffVo;*~GYvsUvHqBpkGa=BU;`5u50n8>2rP!S7!ez^$waM#QO6>pbZDhw zWEuRA*UQD0jFyrGLFLepgFfDOTJhs?8>-az(~f-^E!|xwwX(3VptlFSu6Wa_A?G?x zSYLy1zq81|duX3QG~-HLlgC!Q-rBkq>4s;M)E!FDa-4I#(Y}ZiFWVIG#*GCXS}rUx z?D=EANe4^3y{c;+tUw9==5gCE$Zdl{Px+A%pJns>DZaQxRgGX6k5^#5=kp>e)c~Pm zQd5NRqkmT=Om5w=S`bG++VH*EFay~Cmc<`lYqBevIMRQCd?ShWf_v)P7PUM5$nN6lX~2~A0U)Ogi!7kJ(r(t?JcH}hA&`C%)>FYqkzncT9-|N4!<-QOpP)3;r+VFE@F{?RZ1vtvP^c zQL+@NnwFB01*B*lDp{Aa&so}CT{(z|2GNs6pO6wJB!Ux$)XMQcIx^Kgvf-bJFKmGJ7ny}T&C+K!BqbD-ZZi=C*=^9T1EO2GB zNzZ;+p;3zdC46$0KD|QMdZiwK+?!aqT}i&0s&P5|b^F!sdUW96$2jd}XYb_5Wn8o{ zJph<@U`L-70DtcBGiu(huSm82p~ZmY|MQ(^<_`2Z zdy|JZ$0v`w0lUw{wRvz(PZcVSoC@pl_JjG3PJH!1XGbr6_V+eFqH9Y)ODCX@kB4I) zOOVySaAzh~)%}Q#+Wf6LWJzQK`D2fRY{S~X=U={M1*e9Cy4z^-vfmFW4hDJWfY7#( z8EGwZh@ZO2`bWrf2Tlh%j8AHQqV`R%dNLE4Sl(8P`|o+9e}H`m2DKmn7z(yVeDmvd z5OsC7{To|CACky~x1Z+7D`j#L@nSIE`(GPVXgBDe2Y4 z(P$|BFjRhD5z~1bB61(I5YmQ&^btc6DT)|iuuLFu%J-rb!S!}(nWHdV;xjqkKdL7vJ7AgiBx_zu`*? z|Hb83^nKcpr;%u$*@A96`i4o=3gI%swK(`ztyhy2cSG~mf=+B7J5G)yW;0QJ)fvj7 zHMeV>JZ6(@BsIMD4H?T|Ujc(?Ty_m~QnWDGZwQ7^@>0Z?P~Kj2oOX~Nqu`wiOj3lJ zj;Drq=303@G1Ba<8;8tmPh=z9ApH_X_b(Z_QYcKAia1F_IYPrgeLp(;F~7Z06f2 zW|!8VIvjzT5V}Vo>*kMF0RNH-`?Qq0vFJAV)*hH9;pdlGhd9i}w{J*+&jJ?y?MD`C7WAMax+I;b?@FsnUs&mj$7 zy-{h-M@cktc$x6+1^KLb2)DAi-uQUGQ(EZEpb5w3yfC&yinMcQW~HhakH4`|Q$aS< zg5HW-$NJMx4(M_tU3>N49;>vjT_&wR(C^c}6){0AO+j7rvq|j+x+9_;d|suPoo>2+ zYH9Fwvo+oI9IuPVnl&2q)Ge>kaVx_=enaN+eglTiR-+mv*D%|i^#pKZ5TZZ;>Qs8+ z!EQ`1uTY3v1tA+Wc*jvv^_|AQQDAT2IByJoD6cMFK3TYSy9iRQ7=lS zl+EB7+Y#)wp9NJ*uWa$O$?*K@K2pi)Mg5i10feeZX1?s#ANbI19CSxmgyRPzge~`^ z0Ge!%-Ha!zZ;$_A3>Ht|{s~KWXc=&jMKDdVXRUM+WF+6FJebJVmA%+{?mxa)9%wgl z6FcxsT_*OxaUO^V+)k8~J==b}96VBVf}DMLcsRZUmF$etC{$+)HqiLLDvS>(`W>R? z@Ll*KND*JWR?lw8OCVE=_Rh$lJ-S*IgPet;&L*&bQC2DLGjaB`&_0I%L{no^OG|kv zO2tkn>ypHmvr7-RYV~}#+Mo`-=q+SKLzUx8*Y^_o_lhM)UMY*tb!G0y5dI7qy7?V>I!W-oB+4_W-M_z z`<;4(UI2ooIU1*OTFff;{3{m!fIK|=Xg?zCo0fD?gNdS$eq*21i`}HIvTAil!6)Oi zR%iMeBOgi=YIJ9U@8j6(xDw(H$Zsr)ilobi42R%a>F<)T^B%_G$N0sM^ZU0^LlLy| zGpLSr0#IdxND@gSq3mvx&M5NZg~ir9%@CUv&XXxK=L9-HGse}6H_u56ItoA3n(d?S zar)x5S9zjqVgJ2;Zgu#!UB5Mjol7xeT2Vn;^P4u`YgxV$_TAgFl_HoY4c)Q2NY|bF z=J0i6A&-|tZ|}#n#oxo%S^7k6y@r2l#8Z}OIyG4-lohWI$1Nfax-2z25HTv%k7A7R z!h?Tb+>9Q?y1$VOSa@Oc#adAXIWPz=F^sN?(Sm~%ck*<_GXWs?2lmHh%}LF~Q{eP%1K+w6XN zzYjDX80b%%He8q%=nGXmo%KO-ny1zoFJu1OM>3yk5QG@-bc&*Ogxvm)?%?z}rFyKYI3}N}pg*njto%x}iE9o9W`Hy^TKoTow*Rlf__OWS0tjEiyK~oHru&}!+q7VC zv3EimS|7iRmi*l8rMabpBt3(b@<@aan|JBp(teAhX zpM=p7qos_QI;SckD_;#LN9(!(Fa|lyl?jQ{;B8ONCnxK_5+Ni{${Sets805FUDZM3 zYS022dQUN>4ek1*=@{Mor-`5}2J57DmOS|AlW9UHVh;^zRr{LD^_7yWXjNWcAZqZZ zROB=x1-dH9Ky_UpYP2<6iIouKI8n4>SUXB?yZ;eo^`sR##FAHU;1Ep4RaJTw=@(R0 z7(|++s(2MCz>N$?Bh8)bCAB|58}}MVsUJB4sUiiuk>O~hxpS5J&5ET)2?MfuuW^+6 z6>JqL@Qn;dBh8(w)PKFw=e4w6J%M3M_`mp1A4wylhqaSK2OQ&7Xl~#~*}>Nw8|dlR zLdo&;rqw@_-haaf)Hb%u|DO^A|V?}`uVYO z3iCha&Y9CcEq!!XEB`ZP|Jw0)ddFXE%k51kbg=4E`=9mV@0JZ0=xm1h*&u{1x4N{I z=)Ofsv%HE|ky0o7FBnGYVirp7(n1OD&zcq5QKG}((d~SVCVnZ_mS_g5f)ZSwv6)?^ zD%dJgfEyW(Mw&a<>)rm<`~|;puiH?{BS)oGq`)^a9E~)0u2R2QvD7GGKsN6+j#9sZ zts(`!k>O~hxpS5JuPuFiYki7W3m#wC@w@4(JOAIbaNZK{7zr-mLt{`6TY7a?$nc5d z>B@CCrVqaD*Yrw`cBtxA-kMKp)NLsFky8~Fw4E99AEtZaOq2=j9zUWTB^*}D7cY6( zGtFx_!Ywdz>#i)wv(R4c-yelqtN+i;@4r_5r`P^5nVY}A&o=pcM@i8cwHC=b$%Z!6 z)|@a!eN_OFap>4+DN&dUtCeh}vFcpPt9Oc!)_=g`78!<7kyrSr%926`F?_*Ao+`7b z!a(v_-uUxh;UCTlVwDq4Yg=R%Jx3ZwfXse(*f7W!#%wcCRqy6yU>4uC1Tg;4i zlZ~hU^k8Mh+4<^Qg0ilxND4_WYOVshaTx;AO6Tzp6 z3Ba`rec>$l{dwPv%dkEby-^M%`0~Z>q*zV%1|PST@8Z#BF5iO*C-OQ&ZnOYYR<*{c z<-Y11njXG!sL=9N`}k4A)=j{suf!Jp37O2ohX z;!o;p3CB{u7D_52uAk+i4j7*vN!wQa!}R`lyuWHsd2OxqS@uMNYDKTXgh>beicn7S zE;Fy(FlP@BOK-SOarKdlka>)et+?vEl=<(rROB-u_Z!R^N#<4U%<^}gTpTL2tdcpk6iPYl3|=1ng13YPbywQi*imzm z?>2N{*iq^Hm?SH*1*3RCy2NoX_%pd6)J40nmCClcAS5#=$?9g=+5Q@kVlN*ll7GI+ zCStRTX$;g*&Q=>7_+OcSc&+VRCTIr8S7CsfAF9HS+fvn2rt{ny0f?56@<6~YO^H;E z0bUiFxEzmq;h!0ca2a*&KjbR_=z0*XF0Zsf#px^m+?()+LbxZ10FdH`de51HwoN2-aytO!EL zD2V51M?`rp*NuJ@pTj4{b)K5Odc#8>Et-h1F;%tcj0$()rGr}gFX?BI7x9G55V#5~ zqmZ$?MQKZ~5@(U6z{^0PthzE_D}PBO3p(qhcKp3D=3o8lO&Q?T;EqT&7jhSO6+2mA zDutxn!^cOg=ZdnkGSuK;d|-wmQbJWIN-`s}>Qr96?I|XzU#-!sX3jCW!JDhptU-id z>Qd00xc%T`yM55p(7-^t>AGzqMUAKrH@;rw_6O4XZXTPAD(>g4ved&`D7o$H>?lcT z#nMHwqeKatYuc*(X4M*N#Oks-HY?1G;zmZJ{gsoUylnq7edOw%48;mxMxmQKHaj3n zdNG0i)<62mpG_o{`;JCcKiI)xYlmSo!s=d(m_Y=EEOH?+GXPE)p{&NQ8qilJ`F4h6 zwc~ZBD8pgGyKx%?!P>Vf=|5NSo})Q11wjzjyu=T8vB*;c8V2?>C(1#=F!Z}$_(?e!N94g}6m61r`|?6Jd`eDV431w0ec4O!|Rj#73ZvTT{yZr;bI%nyiUL$-?`Xr@G*RM&Ju3cR{H1N+h>q!(7mw+k- zX$9ll=B+@Ku#`ESGY=j;t{o*Tl$=Z#=rGqeZQ2;R73|V|Tupdq)VYxi1y>j4SfN|O z834+lj4Nx|h$}NxtaK+lDDCpByF>Q^6LX460bg`98L};^%8r0+-CeDz&&lxbdGvMm zA>0&mVH5&Ke}^23T{1@Uh247I6a*uFf*PUf3z==P%V`5RGp=AU+Xe$u-L#^%;-JiR zpyQOQE6yM7SH!M^S4%RCAOUbn@NQ$q1&*4pfD79e3ri-N>lb>)(?s1YHww`*FF`eP z74%h3RgYh_!Eo{=CQov9S%3Pw2eeS~$p4cT4K4Aj?$Boeib^z@GkS(ckEN?O+?>Ae z`j2V%DD?)!8UVtocCpKq2%>?OWlrU0p7J{`75j6;;%v{}XT3nVKwsCn_@WEaz(84m zLTHw-$SJf8p-ZHM!fINC`mZATxV8UfOemCGTKgYkqP73X!P83fgnyngy621eZ{4c2 z3*Y$jPr869&~jV1gjl6>8S}3`G+a+pdwCA;=-+^DUD~7=Wbt6ii#UC=w zedL~i_onOZvA|#OMi_iF^PJPnR@}}6e3}EL_4M;( zpI&#F7*GHDZ+|w8jO&b&zW$6RXnJZenRJa$j;1SCeI&i_9Y0_P$O}pcP2L^zjHg58 zGU+TQ%d4VVAQTFy8h^xTq44pV+)?c)As$T^$GHPv?)*0q^C)`YFv1*{+++MHwEBZy zh7kxdf2I1Tr7sz_^7k;P#+HYeZteIx>*gPNkS~Kh^W1^7=b7i^V@#(_8(An>V}GOL z3ws^{S-+!ix-XBDMrL=KB5+8vvonyxT<>~_!{Adof93M{Jg{^aI#Zf$=JjgOH`dHp z;jiE&&4q~wMeaD`zPA6JKfI$sq4#@aM0ucL6xhF94}D_Px>4F`Unt4`2W3>&WjNJm z4%(DhCk2AA+kpp3xei78AG=SfCnUwF8=2AOTi`JeHbb}rRB>0-9bD7{d*MTNbz1&S zEIhcRk_$L&e+bKkn~_~r$WX)fP-D)v(gcDg3rn;lY7-+)D$0%0tdxA`R5v#RQ0SE^ zg!Z+-&4Z;Vn4B6<1O0>PukZW4bm!w=NDIAC!eULnh?w^t?erK<+t$84eekWnq8(N` zqeP1*_+J1scW6T8j~ji^urCM@xVo<3Lq(F+-48vUoC^-V!H71`(&a!T7gxyZ3%ORUf;!v=f<2T7=ez6PoOa)u2EWwKcQK`JG@ zW9K)u`mIltX>05|uH6ZsWMrP z>&M~k_va&*3Y>vXfd7=Kq_mfl=ZNd3d-S!FO82UwSNqjw@Ayz2P9U2v zre)~jRTc0lKkvwJK>wH@IVj+89pP4y315i!xGzVfu56!rItp595e6BJEzdB_e5 zG5$f|V&QNoSxo8keus5ZMzMG%4vWMf&DqP)wFcOTGe(H zgUh0U0=HQJ{3(uaQ67xI*0k(AVGPUQmjrW_KIFH1vCq!aj!MqD@I@tE_zB)qDg1Gm zDEY#r|GZiYO)nRS2cZj|-B4;n%aG9s!O-dEM-8(7W9IQgz!U9vP2?#+K~??&S$bHlH#MVuz`=# zhjj77U>g6|e0A|EoV1-nT6EXLyEG59P$KxtH?P%U@cQbM9=xz1 ze*Yku*8b4vbUM;#`@bFkr#}BRmo&Q=NKCZG)tS{uHKGmh8Bw26Vw|8y>E-c?_Eb8I z3eh}}tyMvzbd?pb{0Pi7Y-E4}CNL5yI5Q1sp){my*-`TKoxAlTZaU5DA4oS|x0Q-f zoqJu~`Q;WEXGwJy6o8i>B6i_|$K!h4;I{jBn)&kT4MrOyYxy)g=+&YRZVVy@);lbe zHbXhz@>Q0Mtn3+nh`7LsVL3vV%f$?aH}`2JOnB6V^2WUCb#2uP=b>J(5M5W-NKj}T zg!*Tt4+0zbz+d%0<=`*$qef&HAGq{iM0sk%IP+zsCZPHN)bs-+;Gn~wQ%(A+!#J!A zZSARb(okJD44E2Lw7ZP|(taB;f#M{V_OoDtyrEzE9k(tPL6C#;-Dl=fiTd(*7b5VD zj2%sX@VEb+ovzi3FHARW|B1AA*=F@)ENW2enS&3f|N4zTO1t(yl!gZ8n})lQLeZi9 z_;$o_mIk!o@`1Pf4{6DynXb9`d-UUA zLJpMUAIQR*R)U)#@uS5x)s$}!m1$a-zwbzXhubY zA4Ug{9QPA-z*lQW$yHl6hk2(A_Z3o8^xVjjZ`BFxa|(1{-bUQneq4uVcT^0 zw>%l_oneZ;x3Lk7;O$UI|HE=DV?>rqF#@y`I{-9BwOqncr%Ei~4VvwLWwTvRSS%FP zg(ntbgqDXfEU5SaMMf32n05qx0~Wh(7&y|{20?=p+i1%5KWGRqtdph44FAJ`D&*;o z+FM{P5ZWu*(0)H^%Iz<>AQDn!KSuBaCAUK*;3$hPmIL%%NAwEY4&|pPME+AeP|@PZ zpWX2rY0tCY&_c#Qde@tNDy?0)H7%XL+U-*9bZ~fYdgPf~)8Fm*OqxG$k>sa^3HKw7 z*Oh~5Efn_4i5wm~o^HDQgXxM5-58wRT zITkm>E3nez^10As+WDqOts= zztt^~&ERtXE8kH8IjjI&FS2~E#=kL@`Im7Ak$@wMSW6!v=xXIJA)F=SPlnE8+Z|io zV&GRvBC}+<-1&{d9OaA>{|?ey>t8KDOs+I};b4zcO95d40TkAo_g1Z}zJ}uK>)FT=sL&>0#^p&2l zx&M^~3TVildSN9N22@V~EaLbQ0K-B#hUKXa4D_m;*JxrdE4YD%C*|caVVcxZb_^Tw z>$*&0U9?>|;v5w@BqEes88}=ia%dI93q+vD00mx^BfCtWJu6_bQzl;R#?Sz01xd9> zX(jBS7%55pLyoEexGIQO)s}EhYNrNQ<$RGaL%OLxi-A1(`SQb`Nw+-wAN`~T6eq?< z(iQ99p4P3pB3*IOx2GL@{&V`9`#+Nw4la^Iq(uqw;jQ7pj##8t&;oz#=;=#G^>v(0 z7rrUI@7j;1y@wu6-#YM6x^LItYawM^w1%|MaY8gUryqLTuWRvT$ZhZr7NLqURAE(F z#OX%*it#a_&--z(F^h-%gD*c(te~S3oW(pRB1rMuJ^yhUjaqolg#KyixAJetA9MWN zn1B4n?eAxy$j%3OT2n>6gdcWBrSgsH=xx=jd}xB~2OA!CvPh#!V<$xr|A z8c4x`>%#;UsVn^SPb|)3|EUij@C1AVR+Uw3TO0HZthC_30R{S*_5dj8zz-L?gv#`} zIfG1^qMB5It{P>oEfDpgjU@62mfbevkJdz8!GRNcy7!59T20z)kc(eN03Mg<#+4HU zJXi$mnM04HKfUeK>BPi{ziczAuhyKm^rEzV{f%k!nzw1g)Oh-n+drK~$A-0gV}TbX zpl|=jeH8Se?FA?32=VyDsPOftAAH9r(xSoT>CwHnr-z@uL;l1Q+HtZN1_aiZe(-I- zlrBF18sm+F<}>5q$rt=0L;R(?9(*DlK055CY~Q>=2g?rnRbIaC*V_LWlkR264kMYl(P)XJqf@|rw@_z%3 zKk0LsHcbYUCzl6vM5}wrD`69^nO9X%qiaN645n)>?&jb-cUxPE{EyL`z4{h(vhWMLy>uSNSsB45EzBpPuH zoigE_9VNH#c+~0GQF2KrcPb%60{t%`*(F{=@@Bu95Y4!7pwxuP&Olan@$;TCio)1? zp+g+NvGX$_%N?(Rx(d8#fx-7y4wN%Rlto<}iSTcT@TV}a3RaqvJmEsA1|jt^^#v7A zDgcSH@&lT(aD}43T?~J!SK)!a@Y?w?1M<=|KKwKZdL|&4Qitl`<)#?puQ);A$H3SD zQ$@A5NgWy@due{IYVlz=uc>??2%FJD9JG3<)V!?#)On16P20eEZ5Au0N_3PGIUA- zh&Gufc!dKv>JbZArIbZ3mr*Hf=}Q(8Mo27V*UG;gf1W)W6fsQ|@#Kk7f-XIm<{yg$ z16nBAt54iBZORP?P3+9o930|mQ`JzW2G#@QnQ5Q@`KcR zS``l+)fpu^;|zN@^Nf;at!BK?I5qmHPk1&T9tdg)#?5*{?UBvEzIc~`0C5wnb6G4= zn)+~)Rj}+kusIz@(g}31ZMUG~hd(65v+N~0DU>sOU>o)GEZbkBHiPh_e8GBGD+?MJ zWnIAoT#7jo87`Vb8{ie(^bgFzGnL^4DE@Q7kN(T==@XonIn)@pGspuG&@|fZj~wtr z+6zUZUw{C!zoL4`6iOCPFeG~6TOj{9|3(q`l!IV@4tlct-p{Li!-E9GCQ7LP%BT+7 zC#5{>@8@p$)wFBx*L6VaQoRnttGYdU5HjJjM$TJufo%1Ze1T&+G*BQ3_&>wf-8p*35O6^UEJ z9MP5IRaO;=;yLbzw4=m=DBy&`g%b6+$|(d?N<7eV%0U4P0Nu&R7_5+;%-I>(fr%L& zWy$zmGBi(0y-%+M@XlpeN{?$n#g!R#kmD|B=9AGZi<&RzUTPPNZTUMkj+mqWIKMd{fnlVhtWdCGb4t`gxsIiEH|E{-4l#rQAE{p zER~J4UH~J_LW&n2K*Z_-?eYqT=oLKpC4%GYDwQ!7{laefaj4os7A|;%C5H$pfd<|f z!Z;gTI4yP#D97b+N9Ou+Hp){gSCZTU6E!6}@q!~!wbA^sh$7sygK^6fm+Vkk+GdvB zI8$>16W%Lke1!As%9>w&=|eaC)3j#!CV%p`yeC|mVQi29Hwh3coS4vIt~#Tn zPvb<>6u;GY;laQj5cL?Nejp(X9t{$}%2b|-WH=>qZDmj-`I~`=S2&dKE2DJWS!A^I zJ!3uREe?yhlAUFi<=4u;9e?M*{6oW{|9J@GS8twv);e+1hSmB!unu#T4A3((^q=NR ztzxpE92B9Ud7Al+wDPSsg|z%;azlPZm?!mJ1?vo3Gasi0J|nNJG+ z7Z>!oNE4-u#$peg_DwHtD$UP&?4lPmQ~}q4bdO|)viCxR9y24$j7f;$Uum9FOlCqD zEcq*#GkmCGckCOe2+&Mj3E^WWrWxcWI^;ncbkbq|0j>l)R5{;5APUu=bQ)EB6C4$? z{dcy~&caBkZqUb8Lr)NvLG-x+i?ECl4r!U}kKW1R@LI@4dle!l=myc?3xf!ME>Rtm z4f!y(;=D6JR6temOBgJ+0Gl(#R{9^cR%z*I-pn3@FBNdwoVHrgWjQR%7wTk9=t^T*8aEtPZ>*| z5FS@CQWcG&$Yb#4Q5<86*{h42$5#e+;F2h)$^)2rU(kB~S7TX#R#XAUy#lPVOeip* zXG*@aGgb#JeMDmczDn}f@3=~v1@3$=_^V81Gz}}Yav$fh(x{>6AstYP662~Aa-@7` zxVWo3kg6#6xA7Y?==y1Z2*x;|W{xvpgu@2PxI|N-Au^MbQp$^l2GfuZa}DCEZP-jK z(`g2c)|e z_d;ZjgabbCV5rRdHH1_M??7crb7edtQ)&5vRX!9AXfE1^ticnP=u5`P-R)F%aXNVwhl??B(oc$1D4`*@X}6g%K&xEI zIO*SbJxA>nt>F{L($=+aNk5`ja5x#iNAEb}z|>c?pR^Y)bcZE*qbgq zZ z;a6LZN=4Y&ZMv%R(X&7hoG-~?5X~j z-3Ktm?|+O77E-2+yWgUv0ldCCr3U(+Yyo!ELXg)X!*4NS_if8^16?1(hQR_ML@fIv z2e?s1)x5CV1*vBQ=N zmTAA?A%QGJMSs*6TEWmpa48dB6xYDA&G*8G;G6zev{QR&aFF0h2dz)bf8g~M5ZL4^ z9{vbegE;<#Vje%H5h=s{&R>1VcAG^P%Ror%f5EGS9a!K`ar$4Rd({u+aDVTG1y;O! z5k!RC9?^gg<%%y_3<`50JZ9Ij03W?It#i&k`n{jiZWEoMhH$iik|hc1ns?#6S6{t3 zsINaUwzlbvlKBf%M`xayKb6CIT+j=CpyBaU3WF7w8^2=#()u6jk0M72Cy!7GQgMx? z?%!WAa6;Ct|J9DaGR2tLT%kkWIP2U%HiAvC~n0*d&ELEgqw`86#70YlkhTuB0;Oz1dtCO}H6REnI>p)S%^ym=UH z=wxt5f)V3u4(AG|z-PN^o@OtFeiOf;%zK^fCn~ZKER@_wH@93ChCwXGTgu}*8J>Vwk z<~$PW6c!Sy_FK8;v;v?1;Z6V~^654(x(|!klv($GV6#ZgJI+azS9wh_sSUVseoNXG z*v-u}M9K6eL1e+W5l(6#Z?mL0JQ*cpEwg5L=u<_R-T{+Y_ezVLJ69?dXp=HnWC_6p zP3U<9+v=-a^`=<_*?d`A`~BaORx6aJqNhz-q!Lds;3q}!nA=1D^EAMyg?)1BMEb)o z{X#mXb41wH!aui(_Hn@i0ZH0JpvqA!XgNhw+D<#BcapjK<@C?jme@)CW1JYH__bpr z{Df{lh;pC6i}5Ms6M!Fh%SY4Ji*ArJJmDV^rkd$z23(b_`ybnrp4xlB3qIR6txrqj z=-?aC+z@%LNIPgZ?N=1e>JBOmQ$Y|gtf8tt2R;`&Vk99Kx01p@S&zSHzbm%%Tlu%+ zPcx^Tf3^ORrY(w9x`S7Jmn@v0hURJ4o%K*?Bj2+ci|k4fS)K*9Qb3`co$S8xJU)KL z);mf7kHcKoSNzz%mCA)&39bKN2+dY>Tn4U`c}|yX!^Mw$>wn>*E`Ru0*c?T4?@1>{ zhSL*&@&Bb0ciffwM#rUNGp%g2V#j8*+CSE-_6noWSiY(foGAmLIl%j1-2pqGtXu?( z&QaPw&f*um0lY8}4v1ragb%;K`I%OXe{8I|(p{I!gh63^ zEl&~KZv|o!-2O@ql@te%*290Ci$*T_K|kzI)Kiw>1t>oF(VFE$4dLT~R`0egY3&Ez zomOq!7(N(&fS87F7IEw^WPU6s;o%Gz=wYM#dk50reEnagd!GEFc1O(%zeSa4mVXEX zh%i*ez5{8>)c{e6f5JbaS8}{)BI(T6tGNiqUk*}LFGA6NcP^NPLl>HH?L2N^wk?)8TKwAL7a)-ALdNF z61Mb14yW_?Un~EZQ|S_REIWb)M;s#yVI zh)@Usd*33H9_6PDBX%i>87)RPq+m7p8hO(n%nO?oQ?uttTzzsldG$`}y*vwlu z{#ofm)3OOm!8Zh!30(tmm2|Mt7TF|Q<792z>Knt!!y4MR7`QKH{RP8^kU zzbb9H@H(Y?(t%@7rN{R1j`CtHnhffpm`01n2oLp27rDoxW2*UA=F9l_iFEPm?dgMW z{WWc`U7$q|-gD+DSAF4g0pBO*>i$P|r+o(wtMFvnx^aD4s?TJzaKkuxrSrdTKeS_< z5r&+U<_}&dj@&#Z1)^jP}F@BCgG`P$dS zSL|+p=Wbgb_@E2-jEI{0xn}=ge*3Xc@&_bW{{WRQH@$v+y7XiJeOkR~lO9gcS+ZXK z&w~{x#O@N0S;^cxB{VwOx%u3YC(|E)^;7yf4GRw1eJv-h)BfR?3uo>kdMqBI$cIq= zxO_&=AGzw1o6=w}XO~Q-2cQ0m{f55YK|jDL4_fGPF3{2SUdXb6VFZvNGRz%)Iq!$w z`pap<>MLXSDLZ8#oUgJY_33;z(ENFWY2(`S)h>9-3ni8RLnSH)QB%UO@t2!L8T1)@ zjw`>l{~9flhr7pu$}&Sjh>|d}5LqQ+&tEcH`Nyd8nBo>fYR8{u+8bp4RXa);?4<)E z-kmNppf0T@MI)==IOv5ECW+>a#)g%j;85x~4WWo)&{#1U2k`t53@p}OPLTm3{xoS7 z4Tua`1@%*OV0x|Sl_vwTaDiUE8PM4bDxi8%BA}14LeFyNVBq_W+!$k^xghak&veepqG*L4^tUGB_I9E z|CA2=+t2z(INL$lB|=j$8eEoQLF1g)Y(TLdApf3acnFNUVmNM_VJ%jgNqir>3l91`^0m_Dug zr=_n!!MLUSTKRkEpHBPNj=$44{;=IXzd^nR3)f2)4)LY{3fi{L3fUu|Q~#B$2vMS; z7iXrs%4iz8Q_7vsqTcy{zIvmHwtCf4ol&yUuXUjJizC`Wr=fp4@|!x{1x=bcZs#p* zI6680@q<|f{+r^hmGtoE{xlu^5C2g=j`fLnzcOkifc6emj!)3q|Bwd?l8gSV`X9-1 z(TPzOsepB9)jgzmpdgoE>_nP>^UZ0S9<~e)$yd;8KfQV_H~h$e0hYsA@%uOrjDuM@ z-2H2h{YCoxUH>wjx8x$_X$O!xGG2nF_!`#vW+VE%aDV?mEa1pdGRH<4a%=VP>HGcF+8+&M{k8rNVt8ZE zKj{r$HnO2|&{AB{uuxi|xw=ZSK8v!bQ^yY-*Nze;qfXodt+iII{7I@9wP|5esk#cR z@}xk$RTATB7}PNJY%&zH3J)CQ$q0}G6TCCKK)XqXw5!BfD)ZJ%6?56)D8|ZI$t{IM zp%nBQC5DXL44ZR+>A2h^-6|TiRTqX)dB^l#^BsKkMr~WUbU`eX7=c<~(!O!oYcQY{iq zPUh|X=271GkpZI86 zvwe$%&UlP6e2CtD0a@O7&gv0Wq7QjBMoS;*ukZi8bjPD#&`#1N8o&MWomniQH?Y47 zs7t5_*7>9khv%y`KlE+?lOD8eG90h*0MBlZJ9a#sCbiRKB0ZZPKk`TE=!t`R#YYb; z>F12gV+vf#aGQdlc3K}vSFC%7Ug7;kE$l21PW&C(Vg_I(+KwNj2S~i`dql6XQ(pc7 zW2PPIMqO7V8wMh1?LT6XFfdxxRb-_+b$;uAg&bP{lQCk=%dGWuB27+cq$od|{2wor zxTT&T(fDevXr8>(3(;`7iV}SFru+!9a&l=cV3br*colt5#>QOD$Pj)gG9jTJ0eANV ziW3!@JBP5+=;}Bb3QWKvZONOb!(79G5H_ltCnL*5U%mDlbw!{MiV#l4nCQcVR>FsM zMu|S}2d~SQElSs4eo5$X>d%2AFx7YCEZ`l98pfj0EhH%CMQ}X*;h-%4(~||+wlqCH zo*wz7-%P`g=}SD48)7cg0BpgMWTk?5_Y#pqloAl;Fb5s|Uj{|NSwK~vg@QlG} zDM1I`&p!HLuW?&6&Xt&uSGoKrel%_U{`cyjT79LdUken-Sj70Y1EKQ6 z$M>ecfAHU>N1wUZBKMOd3g_^>c|{T41=QoidM2fvi-D#W&q^j(+C12?BroB zlzi>M$6UH*<+5s_1o{eMRk8eLjoK zB3+;S$}xTM?)?Au$J3^Fe}@l&)%z=2ERp{}&)LR@%$21Mr9Uv^_CDOrw|wJsX~*s_ zrK6(-6~#yyau*vJ2m;og&)Z>h9CNP$8etL!VpkdN7UY z!Q09uYtlt)kEhSx`bqhlj04$y90`bfw+g-t?5W=Pm|h{Cm#*4yb9&d+KWkVyN8u=x zff%_lE)~cBpVT7JF)ft%0C*Pg+=Y;~d{qljm%CvrI$i_)0Jxwx9;!ib`jV%O$FDbVROG{0Y~v_ zmjN1aqJ_82VlE{J=v=2+wE+W=WdR;va4oE2=A;Z4p9jAEetq>u#$dU2lw7y{GCNp+ zhK_%g=%WasEV=l}myDc5qf5EsWm*Y7GdndtnI8VwC)3ET$5Y>s-nBN2c)(E7Z}1Io z`WG=LlpipbsQ{w?!7n(~wWM8 z4aDGw|3UVp%zxOA@mK1FbfZ}+M3&MKxQ9rybNo?W2NDI<$G zSLl;Q5c-93&7;HmV%VJz?eetpj*=}K!hb*=6BB7*s5iayn(gV&@BHWKp>N)%k1H(o zA`5b*|0C2v$djXK<)RC;i{xWz&9Y17r{Y5ZLm$!^#OjL$62=3*&xFR`(a}*al-TF6 zOai(Z|7r&PZ#-wA{&CxokeV3dzimIGrS1Pb!`ktGO7ma(Bs5@=UxyaT`p9KD{!#;8 zyB!O1w08B1w0`wU8F}H7LPwcXmrA}D+{$TQb7VI`UNGKKpM)+4YAf8c3V!?7 zAGNHOFV|tN+c={nH!Z`4^r-prub{6xf8cd{F-|g+GDOIYZIU=ToG~#mHjy6v#BZh( zk83-x7D~eKfhFpjY2#5E`6JI9hQStK?iZ|EC@E?OY_vM+z=N;M7mB{@EMX|vIJ#!Q z!T;Gx-#w%L7}r9^(hvWL7D~RycCSY}N(Oi;#Q7vNU(Sjnk{u2YGNezVFF7M*K!>q@ zGyTQaemgyXY_FYFk8lZ-`s~yg9GsVa?Aw0b&Insy*(pL9H_szr~^lf*bxCP3o`e{^H!U%Ei`n1?$a z(T)-w$i`Eq%~~i~rUk<(J*8xdM(gsu>i>)=k25HkZwN7HOi*d&M46i zj=S{g4Z3ypdAxeFL3&jQ>Bic_$+fA&W&KFRNa7w*GCMA3=bP3=gf?1rQmZBOt-qe9 zf=T@#W`n}G4wrrO~Vft;9KO|6fV_vq&|D#Geaovb_z#?6GEY3*eEk*a zuE+l({q6mq(~c1>sxTbredI> zIx9tAn>wlwBZQwM%);N=pVt1I3-+h=f6-$o0_laF>@8R{lx}>p4y%+gp}t>ZFjga`>g?Akw7(!STTWmSYzm`DI9;a3v!mpzI?R>Q zrTXg4TlCc%+RP73933UHkj(a+YzE@ady_uT3jCQiVYN#q+eU~}#cA~{lCWShAz3~4 ziQh=0PwYy4^YS%i^Rj{yA01W5Kxwy!qmIY1a@>Wzr{j2-G z6rO$b{bWQA47(i0wJ5UeBR`Thec!wFj0So2+R@>)7~~SWOTyW~$?*I!oCGETPUs!s zL7g%3=XZQMJ@nM=I-h5;vemyT=Ac!+aC7a(ccu4U_mAz&kcXuC`x@2&Uj z)Vso?I-GDtI&aOv^vAdUx(_OYX-h&9wo&cl^=!{a@lmdtg}nFiwv$jW!T> zjsKj(GA-1FHNS!EtmUm#vG41ajN>SrOzm6``>dR>)&?#p)wju!z1L?oPV;VvZ=l3Z@qd0 zH5P(Hzs}TunS2DHv^uh?#0=e_aqZz9iwy^LnCo5o>J6jauioe|SMLr0fs_j^rmVyN za9-o~V)VhF0pwRS2x%Mp0cBJ;oRc5jalVy43w&4yf&bD!P2*Z9;fxXiQyXz8&<#E9 zye4n3D!k4o9{LZ6bHV=8>{r@;9!T}dppI#wWYs_VP`dQHzQeu5gA<;uV3T>s;?Fu) z;{v*kexZ}Z%%m1UI5>6J-n-LhZ~3HlfGo4Gpj8NWBj?%;YVqXf-t~t%Jar!2>n}!F z8EKqT_T_sX)&j_sKb3vMwcG3q{HwqDX`Lmavr6(Sx%fOiTF5yzdRSk{`&hbi!*|H` zpU_z*`b4qE1fHus;TS%mR<3Fu2x43dGRH>7?bkCxxhz-4oMIk&=3cqZGl<*oF@_*= z5%sCt+TZ5c8GoU=G%Er356e;x7T0rzZ>)uI@dJ^k z)uGt?(@oc29#5FTqko>{0or-5c!l$sm-(KV9XU;2X00Ad*uPw5c!o#E((U){G%qU_ zFY@PsaXe@!hGMqsWz$!C+$7l>e#jDMGZ79IPlVWkGpRF5pv#Vu$3O8YEtEWwdWYsy z!7?}3S^gN7=r&{svbxuTfGtNvP11X2RCUqbyPR?I`?YGi{Pu=E& zTK&XF5NrqMkhHgLdQbZ9tA0)n$+#Ceyz8eDv?Q&*0^u!MC>hmB`b!tANNX-SmOk_4 zU)HC#7t4{-XCTD`^NcwLy?^{Y|6cgDw8Ny*#r$%I7QD{o=HAyNW(#+04c`5<4tslQ zpU)B6wC+NMI!I1sUT{%;FgE-D#pb`6htwI;KP~+j1FigHtb6?CImNK_r6QMleh|~; zAVzy+C?0?9_(wR|{=DAizc=R6Vd^WX!?aWjB|ODv{1^LYCFV|bue+~L+5ccHK~7`G zp9l6~u6!PNnLZC33nf^G{QJ_d_3c#X<16GJjx3QVft4nYujS!iP3nn0%kh(1xM87W zV*mcsqjTXr?2(Hi#T1wW-3YTIVfM~d`=2Ed>e7<6WDrh6W(X|Ol`NIUb{Ey+9VG+W zQSwtCPM3b?x64hyhW5*U&}h&D7oJ~)U1s+Ob@L)b2tLY0bxNQ4G`JxBlkfQ>^n>49PY=9pdw{=v8Zk7@hbZ;uB!-c4exKtY(hFDUYQO|I^6T(68D zlP?_y2IqP_g%J>`;?t1h3vns}PZ29YBTdcti*~+F^-t~sH!E(mL6`frZNGc8Op=g{ zyS(lHcKpAj`L7?%p=}sE9yMhYMcA2@Qzf&6omGtJ(D4&iOen49(+yYYsHa5n796b$ z`T2)KWKHFOpM+BGhJ#FYL9#MXjxQT<7D_Zb_~F0fk=<6>Lt1IS;ToL;KBWR148Uu8 z&8QJ|me#3KrGy2z3ZtMDm#|sP7FEAnsqonFn7?`h{pHK_>Wx1S%mD7-?iJFPIfE#> z03;jwga{4}I|2YULCL-t2J9-~0Itas<7wBgd^(Lip;vF_#j7`AT(i7Yx4d#=Axr)? zy22<5x-gwEU-F@)%MBu~viMV6u*&jaHC_O@x|F-e$XQHZEw+>-g9$k`OFsOgY4iKu z#by#ed0~NrCzC$Vm06%git4f~nHR_s$B3sZ6LN4qk@g+httUQust0ytD14JTtar`I zjp+y9`YU=;w8HsF+Y%Q@aVS{u`0_p9NGAk8q%G9r^KRAWg8#F>h@%fD*#^T!&^@Vl zl|Pl%t=gvX#A`P(!I+xvH2}*+LrJs9qFpQS#bVt-ZQ17Fw*`9Li=k>Chw*_BT<4|F ze`vzjnbJQk{Z{_%_&ayzA72OI?57j5oqP7}xBlC@;li{;ui)^5(dR^T>p!Q9GE|}w zofWhI0Eb>wh#g>|BzBbO)tgl-^ycMPD7jfSRK9{1JO+5s!haS^psc)bo5fEi3Po|_fAGme zirXIjlk}wrKAVOH779#bT>t!lO;G%O-*rEqwqA6D$3Z;E2~Kb{I9&DCdw0sFPo(); zFuU>E?P)k?#E(dS9M;(Ex!0(^|kn07UY6{dIM#%)}$-aPRuzm-m~ zP@)|rY>ia4kiQMRWL7#rcwm% z97%t6$ESP{Tfes5vf~5X^fwny`{CmU(}!>THJw&}gB)MZHDO^RgdDh(lym$Qy%NT2 zYpiPAv;VgQA{uxlDs!0opHD!N{=jt~O`9%!yXo+`NIofSV8PYnxgG>?%wi_*O*4-< zOXwu{b>Z-w>imb#xM-8&w2cvzNh`zYB0m@8_U9tEwWY5Sh1d|I1sZpx$2FJ!ZRHmTlE?Mx?&SWx`s$5d%McvQwf0|v%ks_Lr)2my?yrXZiSp=04;$Sq z_7HwOmWusVA=U98#F4jT`2d@qc=+ztf4!jbEf1snoVmvA(&p=BN!IXEV zV2kkubp6+F{QGqK&d+=22V*!0s6Y#3Jh&MhJDx6E^XBwpI?Q!Kiy;F(xHovlgtP1P z7QK3dPdT8^65o8?mFchU`JHs%lYgHU4=vLOk{_NAd&f8F5zg`P(RA&l-<95d_0O3H z#+4tsI2Al(ZgaWX#I6`M{`kmP8quy2@8lBp@HY@#=AAY~Wz|)_60QA@@n^v_#$R-I z>wmTWU+e$j|7uf1*1T3b+=?b4esH{+@x z|9BdN%)OIDMuqLO-a0FJ@#@VJzxvO#Q1XO!l<3tP%qjGB*|6t6DmdvgvM4Kegas}7 zz+FO9C{-fz3N%g3<3SK!Y$0?90v~xo<%?SJBdo4-R?wG&!p;dk?{`8D!{Q(NVA`U? zTyd~*z;PCE0KAwYhsV;?_;tTn{t}StKlIe!r>}hDKj;wGeR>L{^JOeY!B=vZlcaiM z+9k4m-CK0J{4aZs0Oqq#qSthfMYS*A^MK~gg7p0GZE5$h&kK)msebI#h(1v!y&}g* z3nTh>{iW|pZ{PI8+8H8es5m;Xtcz}R)Dex4lA(~VQym={Q5$KmI8YvA!xez@6=`t9 zlcG{CT!QtuDR6`U78*oF@O0>(+{%`IEB|)AY+tBDS3h*q zZ=@9qF48z)kwp(=SV+-g3lHCJz4u}7NL{*UX}bFIRp~Qd`DMM9zu&uLs|TEUR=@*e zV|08tUApGVbo2I)q_xYpY5d?AE#;GpyRz zr$VDnYyY$Fj7%&-MY1ew0aL)0fl4yVlHAt+3coi*)cT)iv;VDE8qrRna6B*OxXlhx za!XiO=8Te}`1tXHkquMXn3I4*o_We>Ga8I=fmAs&Qu*k08~%z^E^xh^X_$cb3OG?I z|N9@&juIL0c{*6^#%pw#D_cgTK~bWGW)M<(j;kWuQC@OO*il%uA0l~51%SY1*W7MI z&taLj?a&h;#r^7yzj~vD1Otm)N>y=Z_*QaDSeFnh@-Os}wXw*%L7q4@$Vba2`Z3Jz zn@OK|J~7Hd$)~-egfmKj@$LayArxSVPE!mX3tlq|fO@I`=JM!2SFG4@in^*7{U32N zMV~RaxxbG=m&L}>wDx-=t(8JGT2q(1ry1^V^@nN8b!#B{=rH4 z!j9if_w4$j4sTszhll?4!iwZKUB;3+A{~>o_<2G4W;-&2w=k}UJ9V25|GE5J)WgvC>z^fsq%w7$D(ejz6?$fCLjfu2}`_In~l;u3JB)Sq3{a zqh#)U&2o^RA>M+%`m0)D>^mp{{+s|1jkjEWSB6}U;#T-QI*-Qxus{);dTq(NrRCB~ zZq{b5KXP@_L=Mj=Ii!y)y{bFgV@5vo-y)^2*MNz-UN2MG^4#s(1pe18{a(!m^1u!K zeW|}(cRbMOgQEx1#?@~~*T40bjGLNd0PT?c7CwC3@!iMeR~|}z3;NTp@&BA2c>0?j ze8FGw;F)-j!6BCPL>&G<`>e5q_kEZBWBVudx$!@%nSp zg1-4ag~@|Mf@}I8q!f~~x+v&3%b!})n?)&!O8x<9(K?)*Uz%w(J3oRV?M zYe9cF`T>*WsE&`$X^aj~{~1J@l(WNHYK{oz)SIoJ(H<4SuP`&RdL-q3i!74z*D!U#eGXW(r@_``ID!}3jymtvSq-8 zqW~{m`_6RL`8TFT^VzQ?i(G-RiB|gRxBe_`eetifcg+DU*@^Y$@BkRPDe6WBTB(Vh zE`7Vw25twnE_qS<2k-c;)IV>92XYvAnRda4u;I`%Mb~SLHDkt_VqRWe`^O$wbsz{H zU9;(5ZWRF}D2t0s4Sfl%k^jr-e;WOJZM8ocF%OjNcwtxC_WhkQYKPMLHD~JUwwB&t z7iIdeD>Gd-GbN#VsRAWs%2VkE(b5|&b-8;pr`~9wWaScX=1O9vgD-)qBe{C10zp^( z2S3E!p(}|Wy3n6cJjcM&myD0AqCvsFZLcApf$O7I&3=7wUWE&SVN?MZn@|!>Er8i7I^^6kry>m2B!p98u zFyo(nIw>wba*sZMY7p1E& z_^9?MW5)K}O4tAHJCCIOt?#Ap4gdGF>%imMSbmXp#~Xk0E3qE;*sGJ2)&J2;d+ z@TPx~-njNhJ#f;)x}iw63`+U$eRyknaSyXmQ@U`&+1hYE@2zLp$S|A!W$BtG$gSoS z4*vAg*#EZvWg&%LphJq7>Gv1<)vl+=)yN+?>O!X!D(%0~zp!OROMVdBp9e~U!MNKt zgoUW0JK`{sl;kC@IQ2%b&J29013%E*u#86Er2T9o3-r#ExyTBoj7<9k3oS4jXU^xS zY|t`qeCfbBVMz%~Z#c_+!92|<(ec3aPjKWPoo&(%)hU;xj<_Q%f61E#(8eufg+84f zvfS^1lE)p-9wl$Oc&$n)4kNjgSU8Bwk!=Dm5fVJ|(!({Nr}`ope+_;peo|i)7$9Nv zp69rBC?6jiNn3CIyf$;)mU^^XIU8*vR+f1b`O`TgLtiElyhYha2}nJZG2Y>$UjM|y zSKtD-vvs_iqLGT;DNLWj`-_kSGAh08zv=q4{=M3xM2-zJN;pYxjs{AQndfHciw*kg zFEfhyFW>&fG_d0#4Q8|*iBjJoe`-W|@NZdkheINVNzFpeu1>zu*Q0Y+U6|gy{sU-B zdPbYB-Sy;umtKsB2D7g;SR>x33SlXKykx{Yg_k;Y+tU|T?T_+_xBo_3vfwm-VP!gQ z)W5SwRx1G6y0p%DBJJIOP-W29B~m>tSTu9}A5Hr=atl=<;f6^xwyu96znmNyauF;k z^HA5nhQ6hNK&1!M+kfbX6{JWr>qaC0M*mLT_J`gxV9e6U?b@T{8OeL%;Bpx*n zvQmEfa$d!8AScD8i6YrjN08jv{-4VI|FM=IF>*1xsimbi%XK{P+IL*-{iV892R=DZ zdx$U(6~*8mC*o+ajO07+fs%)`Z1*qk{*rtF)tT@xkV375`~|6cpf0U>LW5S(*W(+E z>NwxtuHLjz#|U42;Z53uWQ7J$_NKqS{}0p1_#Wxwn0V0Og?vl~T%gY%_3!Q(Tvdc3 zMT-wWW8(+Yk6iSV=`9<7%)WdVK19ht9wFfU-bc2k-Fx<#=ZhIASukJijvZKZ2@lw< zRYt9l>7KaXACQG6e^J1Oej|VBbfbT2qp!a9H)Wv2t=dgI8hCE(VQw5>gQ$$0WpFbHr`ow&TcryZwE;?5I>o9V}D4x9FaAOoG10)z)u5I6c zPDXTpI_vc1>4J4Ins1^_)2|o6fXfAiR!!2S@P5KhN%FPf#OM@f*H(Z}Ky#E@_C{eLBP|~Yu-Ap63=Od^dW?e9lfM9gn zyKj5?zy9VQ*%_f;9gocHm4uVa8{oz9biLBx^vASU)*Q1NJzsTb8yxuy9@>#%!ojT3Qpa@ffp|D6c$GD?#yS)kOcf{0JEwPy5r*5Aph5WUUvsY-C>I z8u?o)G^9rVXv-2u+(It(hV%p4{>7;K>t1cR%|OXE4U}L{H)z?-I+otZ-f>prr?Kkz?GPw+P?;g@-uA4Pbq9qiomM>3l+;pz(Fh6to3nS-1;GXr~#%{4s z$x9zlY%O6Q5iGkuia({n5@x23$$=Of9@ZWu43z9h-OMO)-yovKH=F%^t@r=OSbi&m ztgAZ3mX??=`}jxG+IL-TccE8%=g_Bm8H#+1E@qWb9jcG24_6&A+vGoe^V4Zy=ObxB zuLd=&C%R`Jf%>zC=*Em>KA=#g;9yXWzS*->wQR=zs|d`a{&m?G2bfFws)2S8aw#qB zU+4$j!|3E_I<5aqeQfznC5vwjL;@aOz0la@P>kr@`;Z1oG_#7-i#MF3y-M=lI25w< zH6}Bg{uTNHdX?Fq(k7#RqE0L*6~$Lm{vrv9sHy+P|CKtFHevgNjW~S!mu<4xDn%ln z^7yKUb5!jSsk6Zha_UVsIQ1Vj;O55+`7n4G%<59mjh=|AQ&zYzL@wiue4+IDr;+gR z1JIy^vEKS)iXUI9S-ZG+;d~#Hrszyx&=qcoMtq3^u7sV~af{T$!?ZI@g?7IN7vgJJ zU(vsU?o&h*c|eDmB$SpfLyWw8$&KVhczm>_ zhS?;lpXe`&fFTFE(<(8GK*~j=Y5!BvCxh1NGXCGAZqj$Nn_^ z%>#ds`umo7Iy7<%8S8EZP@WL7^J!8_%r2o4xDYjt6hXD0*lf+wpo+`+6(>Z-P99Qp zW9AXFMp|S0)9aS4O*dZgIkh*|Ig65=GcpGvhm4CsIyq??d0_x^kJ^HNR9Inxmz+g; zmEj26KP&~T_!RS)17rbp?vOr__6PC}{VeT9{*C^f;_Xk?QubM=-t63+o_bCLC9-Mj z&pE>zOS-Sc1llM}v48RE6q^j)N)$G#ge!m&@QhIUgRZn^@33Z+Y<0?NoqAKvC?O^r z?!eW?=urE$V6|N=G&_>@LVw`Nh%%;SRw9Ey?1jZfuA}lFy3`kqc%bBSd7y;Z2?c%I z{+czYiJ0l=X0Xg}`O*AZ>Hm+_|51%te%-2@QO%rQs?A*2zVnCG-|N&H%_Qm89y!cT z#Siv}4+)+V^*B&Koj#qn`^jDVpH08_w?D5`^^vL@_rR``QbDRYg7p82$FHzo2E3Bc z9R}~X_>iOC-Ei)I1q^M%05Yfp4f~`9;DwCBtpsMNeEjYIO1{MT@=N2W--?I=_?MKL zWxMxC|2h2$1Q%~SD`u3)$E}VE7EPdy-l3-dZ~ULoV<}Bs)@w9hmvk@&pcHn5BgsG@ z^dCJ!R{>CasQ>=VKtQ(jztKO75{Y2~UQPNZJF1#?b=LHOi0Z-=*=#B%YYaIYtO6Bu z1P*CuYc^#B$mnyJHyJO*^K^Pn_FKjoorYfWyA`nXBp7W0`+<`BHRAj$(25K|kfwP2 z>(}N^bM-|&)?@l7V#SjJg&TrXE^;4z`7`rVaPjJr`WfX)T6)vcrdQgdMAz~qIvzL= zlvEAyc+eNwka?7bNtgV-<_iEna6N5yOlythfQ^leq=8$1RRbjhI`xJ%#$pT)$*ha3 zTm%~lM4x3&U4$VKrCK=ZO8J1*KbI?qz-dqlPk?ljYQq~@t^Q9*`sk?~jBYs-V{&K~ zY3a>|AH9b01q~eZ%GqHS3DZvXbqG;0Xp&{=Y0whx&b79AwlEFGw6cXLcQRpaCPNnIn*EZRNyq!5O{Ec1ZyWLw?FjA&Jy*f8A}Y5T=T|H zq&KhqF|{{Mx%Cmnz(B#w#8X6YfV>H;(ps(Yv|qZ1L&wY%(@`;PF>3W6^cA1!qfvpD zJZ1vsb>%R@h!EIGv_B1f$({P73kU z3*$7T1y6YftfYZVPJ^90Xxkm-1rZZZ04#$wg&g6Cv|GxgigKb($Tb; zS@Ty)1BPO^VG;Vl>ua?CKgRM~ni3=fC9OPAvhE#kS6fs6%4}5m5Od{2;9TKDSQkA& z!n=BR;=R;0rT=p0FQzTq@6#S*3*{SXZ7wIyc))}FB7z$5un(am_4$OuC!inz=urk> zlnoeY<8RWxyS@R6fBgfz`!%EF5-q(6SEmxu4CF0S7sZK(r_bmb3XNZr~&-Mf+Jx-=<;+e2MBl*{XN#FBYbpJ>sUDU*}ILHYfj10XDU z36?rH4j{P#fD-yT73otj6_*b@%qZ!KFb^DadJH z4##b$oa?8a-Knp#z3HqoR(N2DFR{!}DfKT@qKZC&pdAWqp&h0D2YYZS9HME>5#geL zjwyVQScP*S6gW@^GQ+f~wnGhlB+$s;YLxq*P?<*mUK8yv+dl@f_o**`;_2sPXAh-I zv{%TgW&N@_%(Rdl)8AD47nZ3QA2*|Hi|R%xEKJ8;$z>C4pL+Aq6V7+W%H`V3^<4Mu z<;%WlOz}V3xTeslUaLR`RZT`$6NOMRGXW_OGj`c~g1t)cDOy8AY2X($qvVAh8Yt27 zP5cZMvHXyxw6+$d*qM~$JNlna0Ox@l+3{#u6gh_q4(0c32fgz}82*gc*_5XEc+Uaw zmAnLBt@M$x`uZL{M>J5fTr*0}d*|ES_pzBPo3(Ok9;LTlNH%oTqTe^s~u2tI0(jO`HB67SM3H+0FJ%7&i) zHS!N}H1dD->R&aZBx|-+zNmee!>J9T0-2hnN}!GS)EhUJ0K-_?Zn?1$C+u#uxwTf| z$YiSSv=mZ<<9CZOMH@A(2n#&xAfSK2+%#8fH~nSYraRz`$hXXFj%b&< z9csf?KvR*-fT=B}QZh2Z8fFCNVc!7;N|;g7m6mEo2}^I7CBRQwzSLV*(}*jgljRCx zYIJ!ACttR3u<&uzlZZ|kDA5{Y$%)Q#tku#Np5xZy6{aiYI#puj(H-;&LLTZ4(Kap$4ZmeFhs44tsr!%4t}5iyRp_ zB(d~Hf|65%Byb22o5#NtvW(=g&{qEV_D`i}Uwl$|=Gp1kH@Yt^U$9K;c9*3$t$TNR z9dNA<<1Upvq$RYm3(?FU}a>wA5LOuc8Okufw{L2Gs|jcshOm`Q38b zy3&O@-mibr0u9>1lI#X$B?GwtcDMvH&w4~s(ICA6mmjZ){b}eo@^AFd&G6K2f4Oh7 zX;S~G0p7(6^kHUhoKj}{M;i7$8>MU-6kD#&7~9{~1lKL8p@%fSoZ@>B~@ymS6xAZce-5$&X%U1#*Zf=5B6|kgmGmC#7HGn)s?g5@bPQ{Sw+DuX`SmFETjn zlYK7SaBf4%td8eiKdb>vxZny(-I9W;y}z%atcv;2V?9)vivmC02P^CH6g4z$vp z9w^afu8Z^W!0K3hgqom3+_ifIGahIt7zLO@i?mE17?;ow4}KsYxjAWUjITp-az;np zpN(puWSgeWwlq-Ur8ml3_6I5Quh6>Af(3|E)JOxjV)Vj0wEW;s1%fk&{S~KziFB%t zbm~8cjM%uz4w%svyn1ntlJ+X2Pg(SjZ$7a{$%oR$_q|(1(iaywC>-tQ&0OW+-~e}N z8ZG7b7a@BP8UT}%#O#w7ho4Qq{k5M~opovV`DL2n@s@Pa>#j;`PJ5$rXo<_%f%HEg z{{8gy^AD!M0}OP?q37eE9N~KVGq8AK)3!NsIM$zjS^A-M?@8w_zr?~H)AZPHZ29B# z^v=y1IN2g6Xf(a&(wlWm;7@qw2^+$89a4$xu?Mz}yJ$|ReksX=k3FM-mlx%v^`uSb zotc*C^e3o;Htd-44G!*0cj?p*EZZ3?S$ea^ z{X0IBjc+!KMmEFki0EfK>p6n5+x#$wo(DKVy5yq-W*+LZ^-9dBW}BY9>VmX(^$(>hHr}9^xE^}ue+tjt8Z^1xo5`QH^o;aR zuK9g`wDbNY@QTjp>3alCE|$`Fr{{L<(V-15q`n23k-GXc`70vL0G(#i1jEXn^ELf{ z#gxF{+~K5jHjM?Qwv=iEUW1%*3 zU7#;Kd;zf?2Y5R3j`gkZRK?UIUb_4)UX`T^h+)Wva`M8DS#RdLSzm;dwloivFk<0y zS8}T15TMInkE-$MG~TQ9poh|Tt4HMrF&!kI?5P@--0($cQajeu@w9a6&46ZJ2kWn02aYv(jT6mTsMPW4Y_@&DgrB$y_-(|LAxtZF%k+*7a3O z*Q7I-Z9=HRELgzhK=763e?9yk((PNmqMhl(saHx|mb(D;q^g+`yLjH}^uN95cbv}h ztV0akP#=087#&Ja?Yu7ykL*qxSHD?boz9Fj@(ToX(D&sC-(1=L+A4s}k3XGu>>4!K z1#8do#>-w~j0z|nOh?=Pvzpd)NF0c?abbPd%UDUI&Xj1v85 z*HlB7-##;VjVYVgc#S?nX->x8L$BCj1Sj9ciM2SFjCiHHti}bGdRGMEae@LsoXYe| zf8`k^^4XT({LysYk6ew`*?;I&f6s=loO;-0A3`bk4wIpm$O{~nIX<#ppL%h#_dQ$E zw>oV)^DP!8vrnlDKG3inc=v(tr+@#q|4eDRB=493&8(Fyf5%f1xRHm_PhNdnI%D|; zeMn*NGA-xPza9<5nM)KDW^%s4fb-HBZ~6mXVd??^+=4-uB;~#m_?>rU6jbZ{Qgof%QXkG zj2rnk`uD19f9e3yVqKxkMYr2ART5G8szEu}&IHHSSS7%dI2m7H z;lndL6%6-6dFYj8v;f8Mn`ONl+WB<@-aR2fDWaml!{%c2>I>%g>c}$rGgp@0EV}VS zY5jHACJl|Kv+vfd5*!^iHPgFzQdgZiyaXWBoF8fcJ$W zh)Ft4Gpk6_%x``IM;stHRhu7sN;69K80NzB&QAS{`{Yb9!zOEw@Bx3M?H_97c%|%5 zL%)%Kqkpfx_GkMmBei(p0{s9C`{unlWV@!?KSkB9u#j<&=qg+V3OH6g+86!jZVi;& zqvKJr+105xg6OvW#bVBEe+#yFz65>YbDt}m_yd#KzVm~5Qa^ebEb(3?f@3q+flssa zW=BdIgzr*cDj@Lrb$emVM%*X}tzY5FmS$bjf$<^=9xfh~fS_3(ZoN(2_DE$rIVh4}R1Kqt4Leye>AEU( z!DFCSWPu~M%7BSqs8RO$^Vl#-d6n*B!Grz^nsxx>x{jgz({Tykl=35aX1uTg+?_fe zm^>@AN6BRuuM=s#P!xVe!V1|t0=(3&3LwW#eYd4z{GK1-W(mM3A?E^u|HjW+CNC||NHNA zTmT797kHo*GqobKoiK1&4*=4Cy>{q7k<4!Lv=lA9+46MS{^Fo{-L&>~T6)u`HSc+V zj^Y^|0sX1`pbVTm{g&y}=mO)*#5?I%c}|@DDby=QYx>{jxuJie?0+NwB7sK#Z5&QQ z`}2NJvfoQ}%AS`ERJK-iQKydu%hi$})qAP6Z1ns(6HKw=gF`y?X7dxGcPO1+jt7>X z6MIo)Y(`M9%Tle9ckMj>R28q1>BOX8D;R07c{5M5k|diip}g z!Y3dhC;C%16CToje{8B8hdEHbEKr{H_Sf|PAwbEb>3^jqH8ru0#Z58_y$MxR7(JED z0Izhh(k0@e9ah82y*=7sEu5DOr2K*_r78-ZQIto)U7jVr{%ohUL-lmRPN^jm zghzfT<_D260x&Fr`}w~sW$?!)WjoP*0Peb<7%h1LmdEmK^E*jUn~LxMhqc& zz(li2CgtS#)EmW*X$s?$pV4Nn16#F6$pY1bM*QIdUc^`ogcMY~x2c&|MZ}&anQ!HD z2TO@SBWz)u%TxJ{S7{W1*BY&WS7a>?f#|i$M;@+MD}CgHY}mEFrH+1~W|VAt-?b`Z zR}7G7*2Y}@r+o4} z{vuNW=RC1*2}k%IOcyS{IbCE-dg7*iu z#EcS_8lAuH?6i2%LVu;?^HiyS%8ULTN&Al}z=d$g>15iUhJNT@Q9|At`AZL_>u6&H z$`d_RSZN_I9--^)FXu{b^#2FG{YO97ot9{+uQ#_;y@s7u{i%vw$9lw7i*Ws;0G(HX zD9@7W_Z<16_5`{2p(mv?htld*%hDS*tg&r{mbA!O>hJ|erEEmJ6n&+myR6n=&k6RA z@+T<8$Ei0OB+(LQmRF3hM+pNZIvzOb)Em1rqk>PY2`4r5$@Jv*z1cN4I@(>9%qx@M{gv>gtry&$iz1qj1 z0(Ft%)0{LY@;^5J+w|=x{!$+!7kfqtnL_sssF$vEu(dC}{_LyL```S}l}{VAYPn7H zk@WAvC%zfHK@Xu>=qnKE#OeREOTc^xWe}$&A9h$XN|-xq+;)1ELup`b`k!|DYx*B) z6Z#(p$$D2;w5I=6yQ3|-{n6H=O}t)`4y%`A{m@Oy z$*7!v0-1bP+Cr2#tTOQ2^cT(s;kJU?_`#=*w^cg^1Qf-OL64d%%dSXfxLf z*E$_0C|=00oxL5JPHlQjKog&}J<5-yBH=KR`M{umfM+IMCnQ5J!x@rHilhT`UOinp;jUn#v`Pn%zf8lgp)-Nw)T19V zh}k}4(#wS(Z66PeljCW#EWhC&qX~lXlv2qxTnz*sa!2g5Uy?Ttgudi(s?8Vw>gKec zffCWM(8-AWX&S0yaE)s7*GpFYbb9ZV*9k;v#D@|nwUZj<=!{Xs)XONK??qsCavsQg zlu)Yk*PfaBHM44ROfb+B#-nZjmPFP$^9)x;W#THriM2lseaX3ze_Q7o{cFpDzT*^Y ze_lEu+cmC_3VaP;vT&a4G(Y^p_=Ua9hR`M`_kUAuBk$y?8_}unfEaIlXxXhjLGF22 z#{=s+eT4=}Hl3S)*w&FaIc_h9zL-+AnK#cMqz=`Q;2YpWjC1PEm}aKRuNd7sl(v5A zm(%zQ+M`6r%f`~1kVVt~Tlm(0_pXwktI}1K>x;@&kaVZ|_3-|WYC)AFPeA(D(!Mau zZoVn4d*|ES_4#;UmhCdLMAd-5fd4zk%V<2zV0XLH5E#m=IBx1Yks>#7Nc)UvBly35 z;QvnFdi*PC(fp-e!fn?87~!BO%qTgafs#wk(TtKe{hwa4!>02tKX6m8sr074SMWI@ zKgOhgdl@JxzDd*nWLr{~lftK<{iUGj6RjXrSLJN#zo~z#{&CR%j)9VBLs`+JaW>d> zi&|WOexiY};cS~nI$pYhNOH)Xar`!t;*(UYZVd<6;0pAmF+?Y> zJc1{hAN-89u%4Fv4_aDQvQW;;m2bSzM&8H;3!dsSy}<3b!hZ!YC1Cmy7r|Nt(dE@o zZbXS@1l3u^kG1`^r8i5LX^)Z%nNb3y0B0M_31k(sCQ%VIsp<5Eb3*ufSn?nzhaKwu z)U;XFn~qCH+NLv2Xz9(?TW?FPtr{rN(i@Jf4MN#j%5OnVTn2HFX89|EIN+!chePgg`jTa9Aj?M~Ke-VUlpI@}cwD64{_0O?M$?cS9_=%tii$RAP!Fhe z>uc25#DVnsRsS$ucjX6^j`GOWa%BoG*a;H(o^xlqCMvi#{ZhZg_)C?Z4?g}(dhUhY z6eeA`_H1ox97}K9hzl+7XL>wB?-WvWrAz38(zsqp`_s^GJ2&DcKX=ple8`Yckqu8T?675kkv13Q-){l1Js9mc9Mbba6%4YkGIAPPH^ba6=E%pD$RDNV9 z3HC_VBRaHT*~dTXQ*Uq=y4AO{Y`1(2@drXB_+ue@Bpq^%Sl-l6z7D523Q1;Ad#vP0YQ$!Ks zMzI{!%#!;bd&Va;|FBs&!x)Tyj=RwtN29rEW1IX`Z8cgNBfhgqhXN^OK-Mv>dnB`)UAOM?^@3f zbD)i6Dqlnv_eVj)=wUF|T2wnp*$c8c0bOyVtNIrv#YOc6UNVFIRHh$agm5YH9wj$^ zIBn8quDBUJc5<}Fn6E+%QpgeD^qV4QYsPF2wPZf^ib1%L@gD6m@3qJpgDF)|J%#w5 zhXypIwUzpMSEjeU>5>?r5kl(dSTE|Bax;tMndf((D=}zr8oEwt-Zr(;8VY(pyE!l_0%V z7!5lxwnsBYhJeUYniXo& zR@~4NQmjJ0wPBE6qkma8uh#ab4z*M()t|4Z`10oG3{ma)T4D~eM~V8-Xka#s?i$)Z zm}N-4nNu-JVf6w&3)7-?2V280cq=4qJD?^kcsdH+D+fUx1|#KsJ%8Y_ZCYA#NXEK1 zU9Eu<_yRV}sY@~CR-;!^W{7f<<7!Z%EwKt#0cYjS9h3@=+qBm1p6}u*M}$^h;yU@Bz*I zl2fGra-C8sZ6n%;2!8r##bo%EyGX67f9N*v+o*9@srX87mA{!=E~TEZGA8IylC zu9*VFk7}UgsateB@bf#hnQJ~CSU*+;8u>-tvM4qioWv=e^3dplAi0Iu_)cN^lbRmFX1-n0!`3|QQS$K{(wcYvu>A?oM#Xo~fC+N2-zy)3 z#V`1R$h_)iEi(ia^7bpYsZr(Ku19&P!$sWJ?|Mw9(TwZV$@%GuOV~> z=>PF(|LXHJcfp-ZSfrWkQHd*08z_v9{@3FIL)>@kM^D)$JFGB}x}3opt(U++RKO}( zQN(f;qvi_7@DWF<8bg_qncIa@lmjR!l;3g0naGWK#4lI*k~(gx^}YA zU3(P+3RsUqoeAQHmE=bUKPGuCyy<$K zdUI{+Rk(|R5^YAzx?aAP&#EBy@bVw3-|Z*YVXj`+RMew#n+!MM63@W~+!n1cS3-mys;#PQ z=;NOKEZKus(v3tI6jCJYIu5$X1&SHg+^Za!klW8IX@6+H4gKh(voff*s4)Gfa;dmn zzl4eoy9)KrWu4LU<=Fp5{*C^f%=SkgIY&zzcD%Sd4Lq$`deZOnb$Zu%uUloi&OQxU z`BClKC0iz?x4l$sX%#hqDz2VJxEQ}Z!~1pWjh5aB$@D!+rsjf=`gLUV$zco34{|k3 zdI+2aY&X8vxK2A^j}pqwjFPRIQ8J+!C0(9TLQBW;7nY;_=NL27^#9mOi9({m?oF$; z-2WK>z*_$of~7_@YeY++4r)fpa`q^B$J>Qa2N`f`o^*2#A2|Tl%o5rL|5ST|DLw_Z zR@sU#TQWDW;q*U;N zg(ib0lsUTa$X^7~NndFK^hrI+%Xy46E#ijcJFuZE14Hb)$3T$%251?3e!zp$GXqX( z`k&ChrvDACkaAe3csBlrWRdL;HiNcP+C9yP=H&JjjaT9D`%Y&6d&23=a$CMeU5dAxtiJ!U)<|W~saq=O`jFhSG5qR!d&z5jGiP zMw!`rWAkHA`$QLJlw6@1B{rMnlZ9=Q$cX$XNDE;>RZ&%d#G}5d+-iN^qeSIDpy`6Q z->=PF#m%zC+N0!R21@*z`8y8ug%5gs#Y5zDBsh0mY~sq%YCKtnG1k&R$*rIF@xTm} z&TMe3vawWt-tQw8Yoe$oSeDZh>gJnFT0V5x10)zWccQANaONWA@{y*?52wUSX0rfWCI$rM ziL*aJLd-PubNwI@Bpy0e-cFSLZ{&{z8vQ$2?N27tKSX!DFqoctcBf3*p|tMY)oJ}% zt37z&ulTjCmBT=#&;|P`VqWM@qW8yE_%6 zySuwn(xI7|cmC&`_Z!T0&3^X2W3Ar;MaHLoE#lUE2pa#HAT(W ziNAUeyJu#H=2Hdt$6rCf_PV?QQ?wv9l@5h+-gbT*-<#CAG(Ok!MHMve1)sq+SlhG2B^bdN5YXJ^BP{4%;3p$(IvPN@|dQXSEigh_&AhW)Ql@p&R=*p2dBF2 z0X5_7p8)&0w%xNx%TGR= zA^GT}@#X8nbFgVta`K#76tX8^Q!HVdqzm+eMjsm8EhA64yFnSC&L=Wh-Bf4^a_Fo5IoLT*2jQs zsQrHIF3%I$`4K|<@=XI0^x4Dbxn{$9?LPPW{Le;h+pgj{-nhKnQt~qs%PhSP+5&6s z31XSm&zdAZ=lr@a5CRy{*kh?k?1j^0rMWYAkvjC5WTwv7iwAgZ*2~5joyW?@+NJ)c zK3<%Ff|tE7pjL`)i$hKnMo%<`AjD+?%vCmBLoBf^2)XkL2yQ;{%kx6v*LPsZj&x&4 zy86cg2@I7E-DljCO)ETZ>X~tNth;9uFWi4}R}5{pGo~%J89zTQfX=?`<1XvP5YGRr zK}3-Rx)nOY*4i_P38W#3RYSO1{vX1bi){IL?LdryVLuK~h@=R~xm(AyJgy{Bq?w7sFZmP}s#$mBv zCfb&Y#hISndbrb4G@D6${#P!Q5(YV zZc*c>Q0=Ig4AAWG>0X3?bnU z;eCY@1|X=H{b3u2JJ(kPW|{h(z0+}o(D=vCkE5v^-}UO)>ShVl(f+;FW#IBBa)J?D zqU(lTMXH-oNt3(`MT+@Tng|pHFyJ+W3sKh|@w*NErFf{H%zpJ0h5LyMq|qCqZfLdF z90GV$TYZ3ikDPK46f}I=V&l>pdW~fw$u#AAQVZs{-}C@yEdA-esgL3i*tq#h1Pu68 zHmq@b^t2>=fHHZmp-WC%*!~&l(h@~Xv$-`Z$5hPHZWWI<@!Kj%;OjVN{C01EI@Obe zA7$!CZf;ARg!gkI)Fh9>3^lTzAhy+@{5hI(`gIn(-xIq#a2z?pB#%k zX$F$BrL3Gq>$>1Q*UG{`*Ty<8Ow;#|^Y&TBBm^xyUCKTWrm?2Y>1T0#t{gr|p0Xo? zqcp*PrIOZLvb|ZRJYJ2Z+1sJzNan)CyGYGaL$s|ggzKRZH#6NgfaP(LPCX$S&uz_I z#8t6&%i!1fN$?-pww$gt9`Fua#2PJzA+Xatu{3sl0kL0 zsbn|71N2`}Y_L%Th4O!hdJY8buHar3e&3$SSAQ(I^z8dH8?C)gOWVw5PP^3+D^ji; zeV;iL$(NN#iXc;u1nM4?xJIl2*b3R`_hqQ{Y~}%kHlvP$JTSnyZh<%lRaoePRJ;yb z>9K?lgV?We>@swFeYndQ;g26Pjhedf<$`y?7LN~KMhIr&{to`^Ztz{p)3UO8Te_*^ zv7gYyVW*Az>xTywZx7(>U`pjc9VS~t1$T&ObJsqSrkP8~A^YHWc~F}GMQ|cefF=#6 z_-Q`Cz$+b1(YN^|C+y~k-a@aqf1wiP0$bX*3$Z@<_a{TWz_vVP)4>0_4WS2fNjGdL zXT~vh2&2n$8BYo$e^d9%bln%v#F&suXfduqb>wwsE6rP_>i7?ihpVeCTDDO~l@6cr zXmo$blN%P2N$flFx&NEPSm92@U7AnLSr4eNNccUwAi%Iv{u6x`tPXeRL_IX0MR#1N zyH1)`*x`XweE;GqK+F9TgUGq5*DcSi{r+^MNIny4x6Kj|2FwajL!y}8aE zC5>?lVbNk&iN<8VapiJ#mHr*Hzl^HBoIw_&9qU~;IM8t>f7gB3o9*fNt6t!u@nHo} zeJp2)u^E+luDb@ZDttk@)KI_B@YHlTE7ywOkGYY9rXse`@z(!27rCig85OA4ZEpT# z0?+^Q&co`yx$wK0q@pS2EJY<8<=lVm{>f`u)NDBrMc6iFqA6uUs%{Qe$}%sfq~Fh& zfLGYX)1u#^Tnpog?f_aVwXI}FN_cZ_1I{_g7UumUb%rM5)M8m7;e{$*vs{DWUN>~@ z5R3Je=eQSi6(Q?$N|iN#2_;95^Jy*@j*g!5ye+iQi&|+o-cjQW6Wt)BZkdx(!VKjr`_ABP_Q=D*=Oubt@vB;9FGFy}{yRzRU#BQ zY-l&w6S#y<^Y8g)AW}q4^6l>zcfYq>a;gB9KCHuZ<(4v{JC>6KrXQ|-b_pNO^)91V z+(t)!q-;`AA+-LdPVCalGbGI@mV)&3;Z7NQhC5d^R=ZxT3waT=mf>R?A#>Ov?Y zU*mHRob4bRCs=q&zw=OuRuhe5nMqu(bbfe|_}#w*@(LILDE*Fdi(di^RicqO&=eh> zoDt1)bXOW;<*J%}ez&TR1{H%IsFq%eJlXpR#!HFoLdcpUEt!6LX+Q6bnfT*eDCf3G z-0Xi~t~!wfgoWz}H#tvzYHtL$j>-D{kA#yK!A z2=4+r3ESF(LzfI^DT4jDdh_*H*jk*_&W5!y&i>jTO0fsD@tjbob zeV8fDGZ<3yPx}I24|%zjFhLCGGZs6~PUnG~pzky&j8DS_Jp?gWvPIZgFOP=c_)bV_ zSGk{AntHF+aUY&jZ~B312i?v?Ee|x(i6r?zHIz$WS=}FxXP}aPo9l+A9zG+{B^(_c zU4qws8+3By4><~^1bum%55GSqH<@BqM>TBPG2pmOXM)qEM4<~`ags_ScD;oWZmH5` zu^yELoeYb;yCv|pGB6@pn)UN7;JU`Km={Lo!DKf70Gh>FL!!x-LrhPZI{jK*TqDEe zICYxPlT*-U(+L$GHzz5PDb~qKwWV_sh`u)5wobS?eM+Q8<6DVWvHX z*PDF&je(B4v8VLo25Tz~4T)U&a%mQ+b9fDg0;pqhIA2&hohlmf6@}hnuL=BF0J@V} z%Bf5(3m~uxz@=&g3EsWUBIPD@8$YBqQaJ2EDyN6Z@tbz!ctN@V356MV7W-ZWO+e88 zZ-|4=M#Yo%=vW>AJ0O>WM-PoYG#0Q6E-t3ong-Y0bDpX?8Vi8VP5BaLp_c+Pq2`4d z)h~)>uR?hTIJ6QhliKe$Fvvoh zBd9Xeij>Wu5iiT!e1ASP4iOjiPfF()e`lTm%XWY1JR$*M-)(`~e`hJ!Qc9Khvd=n| z7w6uyv&vKSIxR8FUCAHsWhP~mm{2g)?|lA2QSO^&5(~`!0zhHR-A8yMs}NKL_6>6i zRPS9EA#o2+WL2+Vdq%;U>3p?!2SXPL@;SdxXq<<3!P{)JF^{?vR%*F-C~NtCys*zo z896l6%8`d|#l|{q>tkB&NS#ki&K9Jx=+HfN|EUFWK#}b@v+igV!dw-gnzC|LU#w4= zH9I94O9X+v)W9=3Aq!hKQHjTJ<^)1EQH)2jRwY!Mkt8bU+y;hOBsofe?Dj)eJIZ<6 zQl(|AfY1-MpbGfFhnx_xH*#74!0%neceV9TRm=GLTR2kM9{`QKkJ~EY4x;GGe$x5K z3F2}J7jNyJIJ1My)c<$vrQp@zgz#?$|J42a7_3Jf`y)mgT=j4)|4#$hc8AdGa8s%K zbf$VJXM%bw86u4YZqX3#V8Gs{=;rsJc*OA_8X zpYe)-R8}!#5$E5`T2aEo{T1TVV`6*AGz)}86E(c{m8al~j2NfoiD&r3)5b_!dItHo z*aYEbISlHUSoBUOj$P+V=OT8uxWAqlql0ox@8%|7AyGDEpoHv2sv*Q_HTU&F40_(K z`Zo4!DQ!1;svc9~YI(Rh{T=)Iov&JnoH2g~Oanm$jT*$xMXj6iXmV)9HCcng%~(`D z-9&D!xT7KakEm8o!~aIzhALK_;G*lK&l>t%v!Kn9^;S7E`J$lF7b9a@ zJqrE|erbiAhdM09FZu1NAB;j+SX>~$Mk!&ei-@S(k}mo^d3Ngv^fWAQmoiU!IEDox z_--BPv&fR&?NpM5KP`8mSpb^5|81#1hhhCm^=#`0d;c;&ZJqnc&0c!fI^LYS=%^

    +6c#e23=Y`c@R_&+m5T*k=5B zd9Q29_NBWnq?}#TvL0lxAauJ9_RKK!KB*H_oU2zua>-k|?%o--QR70a6s3QxE2BjY16vDe8mWNiUz zpZ#}z9|BleMdT~04E@JyhK;Xkw6$4%@`K^DpTx8X%d+Aa!9#vXZAeVTfgeL!M zUAkOYhs{s5`e2IQiAE{t*JG6DA02zMve?hf{iMr5OwK*tkxvij8JxNun~4&t6!~zO zS9$YsFV%AMeQVkq!YbDa+ix+y(WW|fm3#wy^YrAcnd<@P>x#g0Rq=P#VlWGPG4f>g zwY)2pI1;(7;w2B$BTd@UF5$9_i0-*N^Zc6YU1$_W!I*0Jdz0FYqSasUrZy4pDWS=S zZj`pQuTn@WC-Ky2xitqVFiBw90s6EU3MQAICAvg@*XeEX5xUn ztFt!*T{gXWs{iQ6;0C=PhIf_%#ZJFB&+&sCjW z&?w{!>O_34@%O?5&NC2^3-(VDE!Gq1yGTnYy69oo1Yi8?4|*6Y3gXZ>dlU7@HZnOy zFQcu!;4Z?&?7oH2pzDyH4(m$)uaMKqzk;Q?CAE#NMXBRL-@;yFzmVyQE(eG}koa7} z_Vpg~pB4WT%l-Fg>6&I%G6yhAeQ_!&?eJ(lAy0&+5CFdHKZeDIVHCw`PyUwc@Fw z*V<%!_Ziy$w-DSE!h?^jF#}Q>sCRDsaU=Y#hW;13*nq$Qnzm0;T{;_3k+{|S;N!Wv zi^Cm2^6r)DEeJ8`Y!8$>P(zm>>v_MG6iwtfXkN+010*Sn&+o8aD|JvR6#m|S-Hj|( z)LCQ%_aF0EhmL%Jz)c1Q(hm_GRzjp0=yz}zJ0adZS+FG6DL)$4~v$*2Z`M6Pbhzz>Me%Zd`>-|6n|G-^3rzzE;e1P`szr^#mpBOr`O?qr>5Yq~+FWtEN?MaRGmar<6JiuHR=vW)#2M#3pb~`0GxV^CL8k`*dHE zYZ<^JzQn3WOB0MoCqn9Y6%k*hv`Z!$K>fk6#v0f5VogVfgZWG(03MZQXpf}PhUEUW zI4lBk+@7<1P}YlG6sDnK`T9LWSUPS%s$|O3{j}jAidapY=>d)CL z6f*Gbc!_ecou|ef=t=C|rE(->lk3Q%3t}qm@-SR5a5BgUwf?1ov|CAl+w%#6g_C;a znGAXxGOvNcxN9Acc$2^Q_BX9O&=>7clFw=*tcB(??fc3D++%3_A#7s{f%B2ygkGWi z-3;x_#kJYCt4V>kKjPTZ-!NZNTit4GvS3aU8;kLSU~b+XLk@norzukIU-Sx;qnY-@ z_89WHoXG7B`Z+x7@b8pVG_#D|_Y`Odyo%3Nr9h{X3&pHLD|ZNE0t5J-BQ26BAABl& z%)HJOjt0Ue*bUxRWHP{R( zQBrdBJ&itZ3Y*2t6cAYb7!=KL4tKrtOpEVEbyy?ybl- z&Ao$uTi*D%V#rl=c@{MD@wP|rZ^oUCQ7`hsrelpkc>v)~uOX=e>mLNPY{;B)1SUM?b6#bmkE`#eDyDcGRuCoFnABT z`zJRqh+ZU&;9miAj!FfaR{*0N5_cBR@-yGWq@CD6((?5|Ee58{o>Rq+{Ai{Dqo&EVJmQ{++XxN`~(P%oWtee4|j>c`yGYM`je1gv#`8`dlb^KO& z`#%TXGRGHUsi+~GMH)kYULK6cPq*375i{wN37=YCKo)XelfxnjP+W3Dp;$DaL_a{m z@1)`k^idM|;A3rj>J9TJ19z4%nq_BHu9J7C9!DLq#VBtDg2b3)2VUWv_!djy2BMR@ zG|q#wVN|Ng#qPa9qxp8@n>%I|=Uf*Acp0q?cjHY6XPYTa9U+2JC>Y^ef_|N+|G*5H zh@fD#L(SVcO;5@mvqyT1EA161Ri3m>`9!B~cHnKhZ=BSRxp-`B{dArp_l)63ciub; zn;#m^bDkZV4Z*TgU5O0$E4JcK2(FGr1>P|o|?L2QZg0Pze|EP$Tqz;`4NzC z^xY~?y3z9+K&KK3o23dwzuJDHZ2o$YC6Mlo^cpNhT8+6g@1xNVSMNy_G%cA7E`>UU zr{FKGw@PF3!c9LrYRF1w30&RIQbVLYH^hmQFDR%qpXdLqGXXTY-oIuBO{#rOQ$h~(P{pM&sGj=O&v2*WyS2J_=?(TFg;_%BcsBL)Q zlkz^WTcJ>mD*xho;Et6qIwR=V-&xm1Xs; z*EsaN8#z=uzC|!o?6eJog!mXViLWVs6%-7&ZYwrcA(7CehI2c`DqJryUBe6;ClaeE8G3E2~9E~8QuPug=)YjJn>6EUdRKpWN8qXaZgjXM` z=UoVP^=$_Py`v(CYAkF^jN0aUmx{t&UOAl1!+GYJ!s`{7f=shY`f7zB2AH6P-&)h@ zyp=LPo7yYrLSNen@f9?QwUDcm46)o{n`SyFf^$=)||Ox%uwHr1{TAc zh_#)|@5|hmn(bXljIlo3ha{+WNvr4ty$32@n4Cm+s~ETPbmWgS)BTLGTQ#p17JAwZ zHOTMI2*E_@A71nssd+(9*_`Hz#l|Tv6}xTa8AM2DI=;aN6i+m}O>o`zD0-cl$3v7z zFQ?fWfat9-HUsb47d-QDP7brlsJIP1#h0aw4Us(2dfD6h*SS0HH#nPfztpC3+{9lr zjPWggj$}XM6-7VQcc=i+{J&3aLYm`Itm{)sd?ipA|Gk#2H#OBWQI*t1{7HX2iHQjG#NdHz81e;n_@|A7W{+mn*H zA`t1(aH=l->Z%ZUB;Pb{zdC{?zNnq2r2jSUD)U~nppO{Wi|<}5)qxO~r3g*2or<+A z!&$&d`A2D9;kh^fIhX{nF1_qnfzQNt@Z+Tog~60YUV_HJ$l|k$ISqr%cRHTQvf(g3 zFEZvJi8XK_W01hK&m3G+-(bCw7HZ@WZ^P)g574lIaLpH&$$z1IesBoSS4>u+Q1K=P z*V;X^DLIZ4$M9C>GPM)_%a z$TV*J^CUS#dpzo%U7zk43YJ>B*}h&Yg&)nFI3gtUf(Pudse^5*Er`!+VIy(nTQN$-!Gqypy@rZ@rd%F7wW5DMtR$Ui-06X63wcvZ5!{^^IaPhmP^NJ}c*Y4Qu% z+}^;VxU`3cS8{`E#sBVo%m?(3^HEUGl=J_B*}Pp4$H;8O6d#}NG55lX?LrI&^{B2N zqXSO>hvDqJfHKIq+CLT*p*7X268w1wm^ znGN(hc53Cb$p6KlV$I#Y^+lGUjp|8usIHVq?W}C@cfQQKd5@TlGaJ?x^ZNwnaw&U+ zjZ;N6o%CHfb9H%Ql%HT_qe+vld-|=qWyheCo(Q_E>cA%vIn_5ommd+xZ3b1zn<|Y5~`==Bb zs$Y)ot{<~mt|j->3YCTFus%l`QK_`~E73PAun}B@vq^|XN4i~ZWYF~`NURO@DVgF= z=%<1dJ*>liR<46yi5(_Bfc8r)9^1!hq6hqjg*~BL*<1q+B@{+|kXI%`VG&E1vdQfk z{OUJFe9m?0aNNoa1iqcN2}k>`ekG;HH+S9%1%;)O@#^&cXn)U_hKi>4#i|<);55qO zocYjDxLeCEwIekuATk1eN!;0iECfCCsLf{kn;ux#%PIUOzbIWbXYp5w2hN5!X#8t` z6bd*9ef{x7T~|-fq;r1{fbb5StJC)yK27(A;#w3nERhq*@0tm+B7xMdF*B5NnkzLK zZ~i+^Z+%}gz8lF!YFfMWVUM=Kx9_xQHgI1FiO_z3GJbv2^^GpYBRt2=Nq%fu_X}#k zAgpunuOqXugL5=|1|c4ugWp9Zm_SB>Y#X0qU!_yWC5d{SM6;)07n(#c)-;5eTZkzw zA7X3HLPyomL1Fg#uHR99)DLPtcA%-KG9ah8Ef%x(c4aqT4oXgNrq!;XvU6NFNnI0B znHfwLu^(9f1#Tgmd32w!XqwKu_j2|3hpKk88Y?;OL)e!Rf)FlM!)nw$eRPHZd)7(5 zVMx+(MSRGUf-vW_T-zs-n8JXAPsbtUX@z<=^~y?)yYrck5b2BMZpz7k72Lq>eAC&v zF>?b>#>XKx+Fj5Rh&a-hx_I8{gQtekS6f~B#alC;JR+`LjX!hWONA-+$=CN{oQwG| zp*}9&X)4cIoiw6+A2RMLM8J+0fsm;b4lSB%rUIH5e8;`xOSE@H{i`4x2>MfUb<1VhmsagKu1Emj%iLXn6b zx<|U}_qtZ-CJhESBCohV*oU!xM)uaQF`_7X6pD!{p}n_ZSYhMp7?#7?Wd^FuDkR_k z#;mw8(y}2Tz*p;;Ib_>{>gOm<*`DB-BT}l|pXmQgdwa>>zTB*Wczy>{(Y5wM-nH!- zQiXLIE0moWx6?#I1Ms!~8!Rt!Bw;pNLAbDX=54{)^rmtt9JB9_y?3itA!ckrGG#0h z`qU9dNtUlBJ^8~BIiMJ9D2#}y|BgtZnat+6R+>VkojxvVFJC<37as_sm|MhiXU;S# zT-8EiR6h*b5kBH1dYw%4h-P{J8$*0+N{)yBc?b!v==UkQlpCy)U|MPHwUcf`Xh&)4 zp7ez^aV(d;dM|h{6pVABe;2kVx_qx?xC+nJpA_w_ymOwW3{Q`Kd3lWFHyJ=rEmLbh zvE7=Re?q|?Pcif0pxzRxHht{d=|Gr^q6l#~e7`LyX8ZTi#?KBA_$Sp0`ORQ-;l^zf zEt|kq>>l&WK^S?(c!B@p>+)N|^Vtp-RTTIZgScH#m2GM^i6VpxES5lCH30e%I zZ*yY3w)|ayOP-b?|7+`yjFJ@shk786SiRf)>X-8yK&$3;31}kyfU--ab*$Z%=x*}a z=kV`b&l`5F6qmQFZdG=zP9ZT~u}={i%t$LqZe4s(_!t}-LfaZ=ox;(}xQcwVINWs& zvh|zCb9$b09I`W?wlri^dZ}OkIgc6kd8V2U#u2?PJ*(y zg_P=dZbl+2Sqb0MnHg1GXso_8P_Drvkq&6y5oJJ&}$7I-`n|ZFo$(Zrm zN+aCeXoEMm;}W1`4O)ipUTjlg+1^~`Jw9&aeXA^R{d-{He?HEq*#oiL_d;fPG~y>Z zCrUKrqCx^_Jmk!@<2AQ-9>QW3t?#gtj?c!@_%7E|R69;Geu>LWbE81qb$451BBIKH z1@ega62`v;#g?2+n&rSD603^WX|EzW6kJ4EgK!jR`ThKLfbNN+3b5*uL4QLf2h!8j zJIfSKRoh!eR3?WmHBnKa65hdTH+N%Wy<#rbPk*8VP><)rRZIK@@89(!|H=(R0)rY^M)a`z>nNJ0#@J<1v@7_6Y_agzcVd1%i4a z5EbBC4v*ZZTb7RPYY zZaoynJ_Ic*6F_8@YLV1GOS$GmA?8_bjOw%8e_D;Eu`6@}?4uWIMciXt8_7ubQG|~= zR7B%TPy(EFyXSc>pA%)?LTSHnr2^mLu6tod;#j#=xPAr@(go2Dh3w^~H$*OgVX)17 z$QJGe|5z#_Ndna2i)T875Fei<5jvJ zPXnMC2$EW7T_6P`YvpOam}$?g6qvdF>dQrc?i;w};!P#Z61-l0t( z8|Q0T{bP3)X>CXu&x8Nu_8bP!0~KXOCoW1E-%5D*<2Kh91y~V!-WTsmbFQ3!91+z8)2PYFw8&GpE zwiS&668k=?eeFV1y!dUbX-U$I5Vv-+An5$1hd0drq?iF_E%R^v@Dhf@HS-U~XABV? zbV*W_>nzR?imjA(%sjPl*Edmm#5 z#Fe=12bR)4DhVU6m<+N3XJC3vzBFNSHe2b@lF@p0r!Zf|oNe1I7pHOjqHVgGlKIK( z&*o23Vy5%#-dzLu@Y(Au5&tVIgOGpPu9`-Myu z3<#{h(^>@_m$`_7cJ7)L)S$|sD)N1D&b7R<{bVKkgiiq5@m(wWRFOTM3Dj{nE7d~gJBy_b-ZKu5Wa=g zY&!M`$e^GTEj3K}qP%ffxu>l?{0V@KQ(9|XS8QC~Re##lBz`4!s{8R5Xh#wB2%CyJ z2(>>k{&iY}xMmn1=P6&v#hTpC`YFXoM|IV#A*vsDrSM4vGvRqv3#b!iU95@$g!(yx z9Ag7Yj&AZu-XBzb77YAOP(4l5j|%L4CxsZn+0qi_N1q^%)#4L#xa4sZi~iuGE3;o< zSEetD>2IHAaZh?|2VnfSfKhP8Y1b3^e5*xpMJj8Y^1|O(-a3fBpf z4<;KiS-W!l5I?pH@x9eRRUECFLe}L1%Tjw`ic<&upyufQ=C|{(h@zTybdWLQ^f*9; z_vu$GyG=;VQEdp;Q{N2NM>M`SwiuCUa-QFamN3v1=11sj)w8{UR}>r_6w4`X?x)H- zA&P;S-30EWw+Vt*QvA?#+iQm$Oa;bQf$L|txgQc{Cz<4*W7AEv;F8_6QzAPLK)==} zU^QJ16TmVVjcn0&uHhi#vF_8Ye~rhQap@#QlpVJ=w#PJjsEXvQit)XT?5HCG2qIDv z%YLfLf6JxvhlDhP;v{$YrZ1Dvzwsi6hssf+Sd%`sbXy$>pvTRO0BmXZxVc$_lYxYlk$mB zMmZ+VI^nr8o|JT2jG2^lGG*uV z-S)6=BUA8J7BP2DDOz&WD|Ff)N;$|w1{zZ*Ax^^fr#P3;GJ;wuBP|FrgAYEI){W>B$!n+w6B;mt3u zk+M)Upa94Dzn~!NNHnL-DvElz+XC=E_QdRdjN3ft65$|vdB3mtqUu_`Q>^R0@sz4 z$stslgY!e-w&n19afKFq|KJ*+#tXx zp3m0#J5w39SOUjCQfmE4^D9JzNP`Yr6&K*N8{oT35zE)8a#O)Ss$-b}#F3n5(H2?w zUH&n(Kh+mawwr2}V=z<_jo3dobyH*qVjlTyBCF7H6IK#b+WVq3%ke6`^JZrd!Y!)R zDEneA&GY`?FTyYtgR5z&TPVEwc4jtn3Q-;f%op|)Bld@qQGZYZ_)qJCf;XiZsCu{G zj`sMrYi7>-Ah;tc&d6lHu|$6FZ4gHjd*yTeoW--TCucW;CtD=@#!l?ha+znam0IKo z`7bi&zCYTw!11-4&e$MmeeKJO`BYaSs+i>eYq1H|5`#{ozIlca2FXFdwfo%(sSlWC z9~X{u%WfD1hU+BG%d($4@o@;J@Ok`aMB31!&jyzW=QKMy`U%BkKp#)oyF=P)n z(&iI22B4ujx{Cl^UMiw{<)NPc`2LQ$^+^MfS8!75VIjqhbMoz&4MQ%vR$}Vh_k+Ek zgXO7~?oy7sUPh&m6q(OMMCt*U=Nz<*kCYgyYg3~MS1CdOOc?_HdtDT8?ATS#?<|1~ zrk}b78YTz){%LTb+N*pE-x=V@bh*clo zx3|tS+fGE7=mS+cW@6o*R=>6;8DGi<*`pf{M}H-5fvTiOKtWk>iS+TPwrc2&wZHrQ zS8tPoqM)*|fcldBkYd z{O`N2#*)V*ze%=y4K?E6_~8Ntz;CpR-ofu?Gk=vEoJzTm;$`-Hzda3#^riJoN^7?a zUm!B3f0xi2^)WD!ZPSJ7pccY4Te<=KgsAa7zuxXG%u>E#1s@+aWo0+H-0U0|>UVxF z`9Lk(9Yx5sHu?T=Nbq~Fc!WRPx`e5YTKS{r)G?F|OnF4yE{G%&vgY?J z$5SZnlN2TfRBT7w=J*eID<|o-sY6n4EO}rEIdIP|vdhcvz z&{|6|&B%CWeLQFk@S#UPOl0492Q4O2+AyWwhpFq~u*mwBYtvV)TQEBt*X;;re8bBen{e=_1ndzxs$hlc3pn{1Xq+ve?cVOr)G! zEP1DFS$zqQIOzR1P*Q=?tLqVF{p7yf|JJj6_fTXtpu z-yLejRWdo0)+#(Dgi3UV&c@yG3Y8@i9a_h|qxv$cW2(wEoZHY#snTXjoU;m>_&j2> z(;rDI#j8YB#WD$ogM2REHmk|RJi6;OfggEi_3t@ewdwvN-PtciAt~zAVob?=o_Ld-3WcRP zOi(SrUi==8c(+3Vrg_2Eg#6#m#1KapT~ST@Yow(9_C`Vr1=ioluSzIeUbj^gr*?a+ zPEb@ab~7nYbj4xe#A1Hx8$V-lRe)<9!@QyyLIQrP5^*87$5WV}H1bafY>JLYd5Q7G?x ze>mmVk54C&h>1U_>e8kRt`F}AZz;T0o6|W}T`5xj_?Pl#Het#F#1 zK9>q4*ZMKB1Y5${dewv?9fU{i>7-oGe-RWsj!168CZLx?chExnS-TE}IS0dmBGI9& z+knvx%)_wK%_cI3CyfQRO$R)wwctb0e!c(M&b~)10$-EK96>+V{)QG zopK_sddN>vMZ0h*bU_+{%vQpH_`6jO{OJ5%t@{toW8#j78txAyLV}~@x&l_auy_6x zH@3@fl^k=NViTAsn7_TXyrf&{ccX`M%UgIxy)ddq)oBB+rddgGza`5k2z31TdfLrV z{Nt9j7>N_B<)(-72%C}mEtv~;olh3PJZ}%R)1ZAkZ|cJ-#8D-SJjV>*JQ~vF1I^#r zvs+C;O8PofCOI*&LF4~8f3<}mh_FyackPN}_$0N5VYiIoJyISU7eCp$|9$QxvK9F7 zOvU^d0!n1*=4>sWFmRrJxH*96c3C~5t@y9h7WJpB+IwiW+%A%Q-A(@e+Cf84Qc&tB zKUCMdZ(GLx(A)7r-V|+u*c8*jYc0XQ>|}rD{ik%X)0*31x^uuI3il!oJJBNwD%%{) zGU7@rGLRAAyHS?uh(T9V0*jAM_fNcIhquD7dmk4KD=%}smuv56oMHX3Ba$~UP7Plz1AnFJlmMP_`gYP?ETP|2%3YU7v0iBJhOTr|#L8lg59K$R{If+B zT}0}Dib78^&zXcA*a?$L-P0r^g=)SMSczZodWoO_m60&&S4uH-vX!}eN~4ZIwHur} zu4iu87})m;A5_%Qe_$GN(8kth?TIJoNNAilRo`}@eshTsW!&xLTFQp+eHbu~3LSpy zhsqF3_`3o&C3DL4n3ZuPj#0YV{(K~cn@N_2Gb)FW3Lo~xpv7ZJzL1ZL7M<+h;aEIzN((rU`bqF3fQsHgp-Uahb`|uHjo8 zr`W+uB7__pP{Xj6XijjUPMFLVrJYQYS@2e6(jPe3>_CB=|z zRjeg}6}};olK!>7>m`~=a11KqlVs;uxGJ;t;veE4%PtzkLUk`B-&FxbOHmB&`Lg9s zAzc|2L(1V0Yj5!STK2H=mTaCl1XR*{?I zqydb`t|Vj&+~swVYor!tyl6HZO>N3GO>ED3m~i9n|F8p|s8t386=7TEL(i%g8z8D! z*Hp#eSU-Ulye`Pi)mGQTt`lMT%YxsMv-A~ghuKdiwD*FpE4x#IjSl>cI-`7!q^#>_ zzlC!J!h6|vkt#@&M!^`Besyapm(uy5WfhW6-3h`?8P;E|epj*(^AC*yf<4???Q& ziZ)-1c4Ag>O93#1S!4g}URmvZXR8MBUS2*c1Y0~A9=5pRP2J=^Z#6%{9{cRj0VX;A zcOg40DfmKa&ghgwTV7)m5410|_e0ihvkfp?uAwp#AK&Yyp|Wrv#PW=-7%v|W@#YwR zLj)}V3->qmht;Vo_{q{(=dU40E~nnd-ljc4v#GJiyeXB#Q@SbbqmiKCDT~JmJ>Q5q zs!E@SIra{(pXt0W;ufArzbnAC%0sqZx=ER@-`JbOQKUoo8oNSL(+lQT@MycP`v7(6 zXRA*54_#BTm*4BLPX%DTDyY41kOlt1mlRZ%(!bx{<1N3<#=drxtAYQ!-ph({rSe_- z33lGo|G81r+RyJ2ZHS!maCQJUXo5ZQ&94O{%+2$bZoaG!U6WiTQ!!ufQ`-F&lSNZX z0{S#1?^k~&0W3J~lWPUl1m4*^ zys?i_NUJ_Upbx%;4&{1Vc-}lu;re)7Qe3p1TWk)L=hEjCF!Di}auxpdBoBh{QS{Idt$(RYNMp7=?2MYy>xPEp1D;J9M` zKBqpMB^=Pd&9xoOq1R&_FDRapS=)r+;bzFqNxBf_w~6tQ#zy#aPW}@TiEo*&hfXPC z7W{Vkm&b3fnSN@ibU1^E%*}CJ2`=hC%;O(L2Het%sVJPg4M=z^bf8KvkG=hCPeUc? zP4WZsuTQ8w>&SE4bY%s$wbp3)p!w2;v3IPU035zBrV$kraeFd9{qFxTg-b;Rd;Q2c z<{V3!jhaeyUJUk)r(lF{5b(Lmkm4r2Xx;G$B4=f`av+))|K(Q9+uHJOwSHIOzT;t- z@i*WS=;*rJ7uCec%=!d}WcaPIP$(1TK304~y3@KesmK4L=`6gWY};rrq9CoH5)vv5 z-Q5Ey-2zh5E#1vX2+|GGB_Q1$LrHgc*U&xG%$(=_&N+VoYq933>)!jf_eBkLMO2tY zR8wrJ*DWre<+Kl3VysU00U-$EC&y@6vo$BK$?$qOdvuZ2gLTDppvah<9QOz`q;cE5+ zX;+9TspjZut}!TJuL6G1V@5-JWM^DjWtr_Pm0RHwszA$wdu5iWcb{ZmKwjB9tFacH zja*2;XM-c@Bsa_M#5+L@{5|1uyb%{+fbB%#n|s$7cs$`InBcprivC4prYD-^U4Tw1I;G@+6;kU;Is1Ae00Tmt~Arq zVO~z>)N^({-w8RNZ7@go#Oxq%b1#sP{p6OrQhyp%@;i&IFhixT z;I4YaNZ1#;487Out$yFa>Pv+2m(1qA+nYobRo53nMJYgO0P}Tkx1MjXaQ8tmt13~U zSx%z9fOpn;Ks+#rqe8)x4(0CxvwZIds+OL<-b+@8o!j4l<$7u!VFXuo2W}>YClLD| zaQk%C>SY~bwHNts3+_$gIB5aWi}neXHjS9rmaT9xcUwX8NLS!r2~`TqOfa)-7i-gD8p>l+t!J?;v9Y;y(smS;%xQBPSvJ$z@WQ2d= z3CC@JZ>@7ViyfETAMeF2f1p4z{z&ypTGc?>cW9_-!mQWbQWUQ)P>WTB343clVkU4% z6=c;oSX+#5wbfOky=#%WB5xhoM2_ie%ri|o_V`Da z+W5aiqKg{5t!$v=3%j{OL%vl0Yyo@j^Lg7KIQ;@d#5>_=&%@B7^hJXL>ig`PAKBi| z`!ZFiZ`OS{V5**O^~4TGq)?oweZ0gy;s0n0#d42S{F!0wF8EO4UW9Xefb&~3}y3ATmSjTrCn~Vkj9u17wWvc ziv8)R5y_xN^v^Sayy-?H4$J7)KYjMlb1FUf>^@!X&ikL=G) z+7`y&=%ZdqKSMofzxClHmr#1BVk)e%1{+pxKND_?BuzyL)F|S!SH@|B>Q&?8`VNq6 zw$X8im3^q|$aF6g(d)m_;;_XcX~t(G>WV2e!)2dWC^egqU;M$*n4@9?6^Ocxo2&;d z=D$OBZWn9XigmAAum5_2jN+VPZD-!ksNxeJLh$}-2v(hcJ&uI_>ubb>-bRxDsU%)| zuvo}@wt-go2m3+mFHQ;Q8=>zafBT0=$hq|zsXo6GVJ%a)uX~&I{E%4a+7H*1W!*iU zIdGVI5o1FfdzCCI?J)X8g7OCQALjwpy%g&k?JiL574&{XoGPy()nkE4c^4V&C+x+}$n3PqWY3C(^RFAj0qbS_W9^s*NM3n+_RR-F zy6f=!G}My*oh5fYW7%36f%X}238OhxB)W3m0e5PfyVQX7PJ*+erX5g3EMu?yV3L{ z<+ob+<9F)x<@lK23X(w$InBdo<9L~K83;2CW8t!LxGFa}+qfhrcr>WWTu^gFl5xo) z1&3VpJDLWc62~aQbgQOy-{GvS7 zrh~$Hu#%q~`M#0jcO5n~1Fd;HwzzQj)u4nc_W zE&rngxwppq6BZXWtM}Qssn!;al|%ef_b0KHPOyskN|s~u=q@BzIQeR%2O3kDu)LHe zEYe=tSy2F%edNGL;$I)7%e6BJO&ED=c?0%fA#X8mSN0UhDG45LD{4{>YsT37p+fHp zy;IZvInNbbHdJ13)6~p>1)DAaX!5>O8iQ-<=L74R@78e22E=q01}ABEWJ~-JW1+ho z%z{|L#K%1~e&YZ9b3e4Pq>iWV^PT}`-nIes>LWVyQV42;&d+$hdp!UrS#z;;+PG;> z{%FK2E+;|GZig#Dq$%uLnVM5K$c=yJyaReuw6Ovn6T`}|e;4?Asj^7m4g;KH(-26j zv1bQiTSRK4eJxa8Dxc_Hj(4vO+nk6Df4_T?)+vO}3l^qh`xFBGcHiy{ znn>J;t)McO8W_06g|B!HRodHUq$Zn*ew|{%+YIDQJBAFH-as}Xixfl2V?)+7_LoPf z8QnW`I0|4l586mi6^kOy%gy4()fX{HxR_2QZ|FlQ!&&WqVDV8KHyThxD_0!UZE=NZ zw32lNSENjXz}%7_4hYOj`6rwRk^3-<5Nc^<{L@yLN;)dn55M z>mX9H_lIebX5!e8%+IJA|DDqY5ji_UsqV4XQ~X=Z;MQqAVFd?n(nEW#nVphz^UdGN zsllC3)mOAh6Kw?#y;3-0DkRFedSe7soNlR*Zwpt21xn4OLKVdq*3p%%wPe}IQ?eEG z7tPUD*Y#$6X33e6#uvG2H$ntLAgm_-J%xVc5GS*?e7 zQ6q}{VpOYjLI&YVv3pYHlNDAmQLLBGV@}RLnMXfv6sdKSsoAg9n7|$09S*PX=~*x; z)HN%?+%wBf5jc=P2_kYgs7V9z6XZW|IG37rL_;?e9R#Z``lV*++Gi;*OyyZ5zYrF} zbfJS?Tg#CbR@zcHxa>knbq|E#vdXvoDd-)|W$&Z8`6QhMt|TihjUP6`7lvn=n6D`d zaU8<8WZNQ{Q2KpWMTSLbh+cSB`9zx4D;ez`Nvokq%?jzN8cX;~Vz&bDP?jG%9D0XX zl0cbeKU)DFDX=LF4;{20C$#o2B2HJ!D^Ssi=!7J3KegOJ3a^%<63sOAW!b6LbN(v- zJ^{*kHTz%f$Gz`@;VYLX`>p#HZ#KGK)~>&vlbV#(CXk?*Zc6N1|Ao!n{1%lnA|=u% z7V4~SVEkiaz`z1%C6)WSj7yo-MYS_Av#Eh{iuNkAJk!_h=@z?l+@f66q!jm^Q{PJNVWOWsw$JmMlerfhd!HCaW}VxgvB z!4D0$dXyYV$_H-ATw(cxXQ9fH0uKf?C{M-vHImx`N;%zWk%NHboVI5s*^{0kUP?G$ zms^tAqSBh~og^(iQ|$%?)5~H8)wEvi_A!eaNI{;?64Wr3@XNAcMH`*UV@TZ(Ung?7bTkrKz4)S$*Nm?@d zHWb)I^X8)S5RMN%j))t7k(|L3UTYly>J0OSFGl$JFs27P_jgwa<-(a_rijM2_D$!! zFkYv#dY-)IlKbWb7^Su(E2ULZc9PGZG7k0|2*^h!ZN_Hy)Z~9o%~#R2eJYQ9!9>-j zNn!(H65!#;C!4aDO!)cBN~w?;(8J+dtlXTt@2C(}=+4WDDwvlX_56i>XYye~DBiL=c+_Z{@ zed}PWvKmQ<-6~kCqIp3?E})bN3s1lh7G|D+ z*gx{`L-tt{+a=BV;VH!yiZ$}~>6Gb|-Ip`%D5nq7V8;J95kmRK9t!c7MhVUeQ&cZr zdnYtP`t%xB0T-07WHa}5-VmeZU^7z~ccmWWE4HCYHZg-Y8UIoDD{}{*jgn|gW-dA5 z>)_w*U*>u#Bq44J;TAgzmwgUT&A95~Nh2K0vD`YgeypqwD_)MB<@_UfX)#Z=eXfj} zbpR+mbSkRvULTjcN|Mbf*`F@U942ig9-#uLJbu}-W{MG}|2~PU5&q(h`FDA12vpA| z9mG)O#!#292lyBYwM(S65~3v|6G|oXZ_7Z0boJSqPCKjR9EE5sgBVY= z4}}DN>`uPpyC11`f5cRNH*6T0VF=S%-xr`xUo(>OjZ;H#oHN%a1(I)M>~<=MyEB!=+58-8arA zP1A+@htJ%G&9A5*Y&69)7gr6<+LRY$oG;NZl#BgEuKk_>? zN}$5>Sr4R))u+iT!`JcrZo}lA6WV4;5rH7D=S)ZAN&eVxBPwpSV>>hZ^ManfEkD!k zlx(G|-@7njo_ffX0e6cHFx>w%$QSLMUa%e4^aIUZ-x@HqXSUmF;u-lWRUhq|nRA)V zhP_#5lMl1xGYA=ipoGAYcGjr(CmH;W$8;UMmBNQoA4&g{ADmHXYtVU~>1X%vi`}M) zA1imM1Hl+sen3*5Xl-ZuT@wB(!Y*mGGNpuyYWZ2qQkLpO?FGfiUx-v)qEhm_Is1W? zYc3Ep^d|*zx)O28RbOGeOUi#^RC`z-l`oP^MXj6R;|(KXiPx$xc#`omY$zAJt}yLm zY7~ltQiAqRrX8tX*zh{f%zu|IyoRvcKuhd)5^1$Bz*k=@L#mkYle7kQak!J&tP&XJ z|tFm&HmuHf<@+USJKuZi*iV(dGV-UkWk$V8A&47K;+S65?QloV-0oCay z3*~{KRNs_x7)H0o&fkTy#f?etv-pdUi_z?w^)?llNvlWw=E1-=ZB?F3yS=>yQYz#G z!{XRB><_Bu3H-lzvOhm2e>GnPAeJAg?9+ zwP{8fJ1-|Tgwp)u{DjZ!)?>_6k{^d&(ch(gSVLe}&{*P{m4;0ObZ8S9m5AWbD#^Vm z2GS=x;i?XW(-`*`)S0X)2hR&)C->oOpBkUdzDK7Ux@w6RgDI7qr{HyXk(uRNgdF;Q znY;mW^(<5{_!VuMJD|L4W<8&hnc^^qk0*p} zY?lu0Dh4OvFa6wj7^4_iL{i{luiQkX1s^}|pM}Tteb9u&w#A|HTTyox0)(srZJGBL z1@=@2c;F$heZnadTjk_mF$g1FmnvnDXZzO&J~(u)0cEA4IWdHP>-XIwgqCN-v zq7(h6)+xsabT5cYLRu`T5o_yk7np`dl=Syo;}6vOyRE1Y*B4gh2ibN_uux>n;Rm{(`##UsVMQbH=WHxuZ0w$vwhxzPrqT*25QKGDH{oO zzEdvmr~4&NIMQE~O}XV(|LG-KeFx$Fl3*-YZ}o;wz{sCF=y!YYKZ(S>3D+q0)APRM ze&N|DRo4x^wkO(yI#g6Rw!`zkIiIpUeIigdm1mQ`MG49p`VWb2GD4kA!B;5nwEBwI z1pTBG-WAy@Fc?lrn`PI;Qx(w&qy0ieRYLz(>#L0- z@}dIm3-T*1M$x~IpAHR+e?AFytY>cg2yqDEQ~mSf=ku$Iv;6c8>y-8xDJj-)oo?qP zI1|LP@nH0}YdyDrBVB>v@{##DiMCWisIB9t*1ShPw1J3J{%Bwl z(lU9xD5^V3Ma4d>P2JGv9M|vvuHV@ipR1~8jrQFe*k~DReffCWgvjW2Wfm72Lvrl& z1Q2<_7;fsW0>g;kePI?JIt9aw{yNTdm!b50|2HL_;&8yBmOrg;5hc--*Jxd5*WT_H zP4--7`?{d)%uNY=;wk9$=xfW6Q%Ub)kGjo8bmcbSzD|i9ye8tKBoXx(vn8z@{~5OI zt5Bi&DeD600X#J~%o2~oO=b7qwNV-^5$T%M7w$DLhYFjMbo(Zk>{|08-f~=b5dOIkCJwvbQjJ34HFjvdrAJ&@25G)v|29^OQ@i;c$vW zh#TF|WUN&G_+=xc1e_?H!u)mQr&rpb{%hi+L4AFh@L&oiFI=@5B>Fg zsgCma{`SS!S$)){5wL0w8Pdo3aO3M?~(Jq#3d*%QfrtjB|P}xJdGO+h6QcDdA zEgzxa&ZUQR-?k9IjOjguxV%oQV3jZ%_f8LbB!oRV%(S|#qSrHuJqrK`p2Z7VZ>|?; zluco8VFSgJvP79t)@vgKHaUvc{J5cc_Z}M!=~&l+axYYW{q`jCIiGK^r$YxZw{Z+W zG;Fp(&m}4xG+PJhS{EXZqOt?xpKW+^E8EkGxBMH@KYKGs3Nq-L_<(pKZ|~R4(pDvB zG`(l8d8RJ)oDPc?{#SW8IFO5p`1a3wG9fa>N8F-Mo2Gw%d6CrbPmRF4v^-JdJCqCm z<*gBrs4b&&x^l01L2V(i!E&p?%VLxO#wtjO$c|T*Fjew8MKeizLj&(JxR3>yw8`dP z*x}Lb+tTxE3sfN>r4fsEej!d;VIS(+p+~Y|=Gku-H~80-b_d2B;DyzGR8>jQvVE09 zc7&(^5pTa(Tk3fhF5E-gUAXA<8LOjp`2J6Gy8poujWYO}vVG z+d3V)mpBgQLdbakN8FSj!wJli(1@+H}Ezf!X}Xu%_G9qug(@w?UpXoAW4(I*2p$HTlv zUqiSozmt+w8!gzGyL>6%mgeOQvr$y5#I`Ld8PZch%9otfGx+f#x<=N=4Ex|m#?dvv zj>BB{3m*{ezwoD6$<;a zw5l2r?J;P}j7_Ze9{TEFrui_;id0JB(U{6yJJ z+WHFtXaUiffaGKl#C}hyxZl_G2J>ZVEB9o? z1fVN#Uaizxy1!?Qoi;a|e3|TfNB~7lpB)iI?XTj(SCQo#&4<5LS1#$yXS9MXf zT|h|3w94eOLZLym+Cmt0rEhT?3s$WnwJNrT8cZs)voV}JW_Q+J=n1s-o+bLJZ}zzdr#7( zm^k46`P=Y?^ykn5$b%==q#eU7n$SG?*46TO+ToTD8NRX89RlE32c^ezBv!_(cY%yB zMdS=V=j>un_Kw%X(3daS)-zsd*W4HXh()Iix@t+DZXk5>PqPzUw;uW&Qkx9`bq#3q zc;+xdTU1m-{M*qe^-OTn<^47Zo8I|+|EoJ1cG^g7HTI#>mPbY%7S%pk z@coU>BRz1l{SQWz3vW$Xh2oy%QWI232SDYdA*R~lk%Ft*+6svlg~H) zk-oA|(EFl|dS#yoQrg?sqf%y-riq12*cTHx;I4c=Zf2BhRbYgsBexWuOOxPE<*e&_mJ>E|ND-;@ z`+#pU<*&uU|9vlJ)X0c>O!{D3uhxH_AJu{2uuDBitoD^Funcp5%J5Caaa27w)h+G@ zSW`?~7`)%#c-ip#fYHo5|8OnllHO0S%&hs}1C~XC!$D*Wrk^6sDPAbqvuGS9lGn+r>y7*4mfsSi4LX;b{31IEp)WFoe81n0f*&oky=n$y2mi?r zfYZ5i3a_OwB%Wl-XF6#WR4x{8E~AxHU~S?AS7!Lv2FshR1`NnI)}eUhvr^4p;az{r z+kpEXRN%U!KB9yAWib1wkuDxCFD6;`y9fE&&~N|PE~4?uI8mtA&DEloWZLCQt_TW8 z+lPzzgi@Lhrx%9lQ|iI<^#v?NWw5k7qOzeI5}=6|cxzre!$z)|+XTjv=^y@;_@d+s z#XyPisn720mj#2IA2wztz+Apw7Kcor+yC^3mijg5J{N|J-FyR1Ru+SR=A6Kr zk_Rb7i+B_5eobjGJ&|X~I0R*xhA=8bW@vLP>-D-UVV5 zlF$qEi-i`h+T{7x+pm#kX`(*(1|VTmCmZLZ0*C_F<`Hk1hJ@=QguU-6nfGhKHQssy z;eG(YWp}GNS|J5-U_LFde_Jqj@sJm}HX(AJc4ib^wU#xMI95Z`bc&``TS}_QsC98# z$5~{#VQ}4*DE2%We!)iN zNEQ#`--p&tW1&f$gjN^(KTP7V@IIk7gg5OwJT0g5^Phf`^e;ENdR+G_=)MS?novsQ zN+BY|Q>CP^qQgD<&5beKdQ_4;3v=+w9=xI%M8JV9Bq8%X)mdI#i-b)_&x!qq3iPah z%9oye97D5x$AufmpVm_pYXST=!<3-HOplLTaz`JP^lG=~1I{SFGB#~9d`^bK zI^e#>*lx@#VM*d|ILl@%UnjoiV@lpGH@jCdtw%BKaox_$AubF^Pn@!!d6rimnbA#| zH*ky>+I-bQcC87fNS}d9!s6S>W zrb(8v7hzJnA3Y!pM^PbW)w;RHfGIr{z6_Y&en8L2>85&kPk?`GE!<$g!##ijx)0eU zi`OSGkVO#vxe%H<9hs$RA8-2Kho-CzvMARl33yVWbdfZxE()*|FN$fLDJrwP++Xec zawcDlflEMr(tai7>UlIbfK(=Z=ILJRPHL^JpjnE?JJ`_Yn1}pSLJz~W~O&%wfO?fNfM9Z3F zlbOeRpI(ltBZsNi3|~fW`NJ@&noybC9Hvd%M&kKoC?}XEIMiOo0PMWG?C_`#Y%ke) z;tAJyPqzfq40rFEoJ&O(nai6|Z%r>1q#uGCfC)Laqr@8pqz9YF?}y;UC4<|s>>4%` zpYXB`=|C?i1@1VXE7U9pG~M(Dul#KS{dwY^{`&ap`8jtCISyG+Uel9b>`%%ihu5WX zf3=KVf}}6@2MgO#k7pkZ^j1gbl3MJ&g&cT>OyA^H&6rInf!S%8s~8~}a~(=@SB9BH z`qIXx(+1=}0Wc)rp4JDbmjqJWf7oQT7@xL{PB9IDe-Pd4sfMu~@^NI3GHyh7z9J`& zSWGntx{Yn^)fuA$?oi*`v&v0M&=ZEDBf74sP6DBC<+P`w@qfVKe&Rcn4y46HRtJRGGLrptFMMC z6xL%qJ~AB7Tl0z&0o;`>u>m1qM3BP5Z#)O^l)|nJBC>U@X@k{nkuR0G+gHQIUoX5- zT%1!-AB51*T9lfv?FnQK`JlOodpr^?!F}n_?P{~D!-nQHoltI?tJB*gguj|z-B(I%bg!FGhp4Tz;td#^ z1qea4GFNDAfdznpN0>`))SPQBqyL%c*7unQ!=L64ke>*u?Wnp^QjpO9B6IE@!j)-l zEz+S>FYeZJ53?1^@PchEi7qT^BIbf`4CYYAR1A9>mEnZGry`hglt7UM&;3xQ-Y~rp zERrhdD8Bs_am(9zvgxl^=Ze=#*QmL9?wk1`HlkgoFOQRqLWgVC2B>lrg7{i+X|45% zGYWEyfB@X3boXn$bYY&ww{sY`phleT*sVet@s{V9jT-$uwzUP?OlwYxSjD0oR0Tmk zkv@g$TZ&?wogWa79&{GlJc~ z44B)H)Yk0HI$cDfb6Z>dR=t3&nP+bg7`YMvE2bf&83bEpcsnaz`|PCU7N*!*b*A!# z%{$UWzIY-62Cu8lPajZk97c*|-G3A7#ZM^Z zAgMY{MqW4&aLIA9B;6|UIgdEKTgAp?`ny+Oi)A}F^_{I;Cead7pH6u^CX1{EgKP{s zH$3*k4FKH~(Ctb+f3Qg`k@Ucv^t}w;_RzUVOC&Dk9Wzis#qKE~z%KYT$jJ*=VB3tS zO5E^lRMCeH23;#+Z`73IGbOq@lt!ON8!#Y3p}(%v{|8X5T3;fpsnPb&F839|9pt85 z2l)h?2P_pgItg7-obngysIL-pdZD(BZ(0X3y_}72p8^9d;u%$-s0W;7JXpo1=-JZD zz_kU#l&L;8!l%_gv(nGh@x2tPGr#3-tYB@FQ{eg|U5w9tR&~lDd|}v?@9l`$%YWTA zD68m_O;k_Wr_mGD%_2UCtp|Jz{IcmG2lS9W!Ilwu;4*ECn>`O6t$8g3=CsNLc`GgP zVph0j?(_vCIS1<34%wRd20r;b0X|{Oo%g@w?T>DAXFcArXT}IjwR;s^ZO~*h;U=V; zWp8s|$6}ydnJpfuV52v4?UDwvYwv%To1?ZiU%}Z$DkqQBDt6!&ubvDnqduJz!GzX@3QpWDz4g+_KWyt=mv6#rK&Kp1xv?e!zHL`wFYVH?vr2AL{TI; zVDNj9u8+XK%l>>uNL)G3Xfh{zPkqEZCE*<997h zgilg;A&+xy zC$+#`|D3S`78irn`utM!*j{&`UGj=7Us*_<%1ug^g1`rEa7&X`qh}an+*fH!hd;vj zu6m*{Lw0kR5H;2298f-~@p&uJjl=}hEXkcSt3Klp->M4wah4MTo2qmswq9C#vQpxSGTWCf0+{sw zKH7zYyOI%YVxJ_ms z5r#6snc5KPd5;c)Y-t7yiJpr|Dc4V6Wd7c>86VEQ@s+Bx>(rLAc#+#gSbf zlpb!+yqC<&U+K4IygGd*Pgj>PO*A*fm&rz&?R@rvVfQe4HTv9{-+k})WVX{csj zujH!4jrg}9U`696R9@{8ivq;=-N(-sqvsvl4CoK_)lOLi6x)Pf21Y7xoz|*R`7zIF zH502VPh$;I0mmY(o8jhm0j(2L6nHU=XK^0{9Q`u;<0E@@e3_3sO2C#0Y*j*%jJw2q z=01yw_a%Hi3jRKG5$2IURO5+2u+`F}DmAV4>9YY+x#r1#cPE_I17&H7z~gHc9i2u2 z!w)%`wOQk76d+l`Py3=~DkfC7q6C%y@wRyw&^I&|oLiB+Fkm_3*YXR>=gUG3Hsqn~ z{0{NDAGE!;Y$j|yQQWU_BCy&R?=8RDgraKk_rBRtQ$AwAg`87<$~%oVKHIwu-EfS; z6ooHO;U+lTD?_kejNG6O?!dfP*vA{dc<-0EuruD-)=MErByt*&vldNnEgpsUa|v+N$U_!XhEwwe^C)2Dx>T2@tTl}zNE$xf~k zLA+UlhspdZPFMNK^G~w0D|Yt<<--|=Z=j2gHxT`-wXA$&m=AE6qg@5BrhQX4FMg@< znS~OOkY<}{jlBSg(@^Rp743@7+@&Kr1Ysr3GG4(Hx8w8#@rK`$v2BVE;8CwgZ*KkQ z&eC4^&2v_a_>%ti;xI(*>~Rw)Ql&Eg#Nf8Ni#d-T^*tPaU*8N5nOKg}dXCO{P)5W( zJNd^u9r4XCOYe|}Jw(e3APNn8#RL2Q955Bg$3h*pKrAn2qrb~-&9Bd%P2V{-obhb` zyzI`N)HY8_-{qgDL#na>cEYrF0guY+4rRhCO%i(6E{(WEi?PqhXC;!`xx?yCiQ?$- zZVr*rT}EztJ?KNWHV~(B@%zCZEB~W8(RAaNr}qZPp>93Mjq;Yy<(n6DeHqIQ$CaC< zT(j^I;tv=K4us!6s+GMm;#p9nq?I?EMA0_E(5H~6y6STWXQY7BtWuo;;-f-1u(Z9> zv|l<%loa76qf>kQX-ADvk_;K7xI3>QWfmY64c9osPVmFbzd1q-mzJ$Uur;dKCmBLZ zwrG~`v!a>G{`bC|(%VDsg}-IkcEf9j9K>GE7gD6-?hZH7wLdVUV||JefWU?92j9zK zu&6JmeXHn2sYoXyyXORfDBN8i34KoYiqBTg1_Hi}4aRrjYa>5*K0J3728iGT_b9Y1 z4L=V9LyAcBPgQIGO)F_V1gxxW;vrea(Q(~Cbq*2UeD0iiJ#Aw3T)%fqM>LYwuQ~}a=jEbi z@E2-mI8P_p?g}{6O`lW@6_vUvPjKWv20G(@6TbZ?_#eq8%%hz8vwWj)9dr$4*;jLZ z*}pXC5aIhmc?!+_*bt5D)W9e8>Ta}Y=7*hzbk{~-9}p4+$e`~dgi)NI#PU#jr++=e z9B-h_?GmgqG@KEpa_$Uy2M$>8>zhzK=dC$_K9=x$i`U8(Nzb|Dnlq%sWRLiZKN0rw zeEQvLnX8~vt`c=RB=3YsMo)sUoQH!fXryo6dLsB187&%nJPEXsFd zF~6dwY(t>?eLqr}L#4~STu@-?8}K~Er9Rg%_!$$^l09ib&ZWw#ALcAz+;vg+%=&AP zPMl&Ug60AW1U<+Jj*9qjP5JxxMH%YHausph2Q`WS;A67;PPE0%$b+NrVS?=o(MeI7 zla=KZAMO&EK604X8}+l&9;D@YDaN!i-ZT86`+vYKusP*#2d?Bd zbV@S#Yu%>={7sfAY$QzH-jwHv=m0w@yY=0h_vkbOT>975<7op(C@l)}!+nm~o4YJG zE=|Bcgwug}ZKUg$7N~kad?~SVh5P6)Tx8`W`*|L^Zr-!##TIHOdufj{jmTzIvMth4 z8gqGAFq{U2Ft>F!$Z(;v=Te30v-&!yepOg|%sVAe_C0MfIz7?r)@x{Uo*(D~L5M6H zba!3^shq$}IAJreZY>Y4lSCWKm%oo8p+&1Bv-U^yFBCX{(-f$Oeu#CC&;|H9!8G(( zol#qQCGFfdDO5G^=$!)9u?ZYk`I5RuBevC`B;gUB!p(PFD|w~}0@&Z)A5MAkjk7b7 z?^UDSQZox{4_xCuw^~V<4^|3%+&OvN1fLs@8~V-;{=98JKIh1q zC*Sl7m_8ebhB>s0@oRkL0Om`Y!^k`F*4~10>%xV+Z7SsK*~L~gK}idIJfvlS!LJuq zGcA3$7w`|)e0;+;_5dCfSB56{orArLTg#ET4zlO`nIc`bZjGL=x_Iv!^%b@JVUO6G zDH%BO0$_K6pfcB4ETOao9c-B1vANVBWDvaoQ;6#Y=3h2$)|`a#0{2PjJGwS>w>viU zL!Wszaw5-tBVOkn`3*PT_?d};;OeJ;4C;4n$|k7PY%-R=UQv)&6I*eQ0!md;@|U?* z+@QCbKiRGk$;sojPv&H@u3Y3ECN!o2JMurZQw;?rT?Jp!d;RO&E`)SM#GQ&n-wT45 zJr75^DNdUO&_RDscO5^!D+-#H``!*>vC!FZ<b7sE{mnf6|TN$pr^_n&mz;h4;}rG zIboXnjirl8pF9c*JBHfMm~KrOLW>Kh^#p8pQMajg=rP+piXu;JUW~o&orT@Z12j4ZlXKRg&PFmXjk=s)0W* zU4);AXlM7{k}oriV=VX!)psHL zq4?gS@A~klv}stKQJTsYQ)J?|uOOFpG{$9Hw8GyP+>C*2>DiXgf0#{uGyH zov%5@!NS35J`ijhnD2;Zm5G~hYc>pczV1@8|LtoHhNSXT{CsTrYgRcBx3{ThRSn9S z;{pX0NrgQ|zA<)YK(|K>h1rCSm3Zyf$$|;jd0YfW7^C(WlA6Auh9Fjr55y>3WSv1k zy68}?cBY`huIu?`w`RP6f zZl3)|NaEcK+gJzHd(xml`Zo@0VipMzhsnB~Uh~#I6}o1G>^72KlqsQM%H}c+(P~x2 zcmqFLDFMm}JOlxfZE-cyu-q*2cvcyu29p)61;@;PYrwSMa^=Eoi_;6jvzCH z^}!Y6FDvYgP^2^0oZo>W75#a4h+swhw~uti`9bp81Vr})OvB9AHK9m}g7_c(krx>` z&AnowtJ^T*XyCW4v&h=8T+HBn=QYuJ2C^uCK}K%YW+VA+Shij6kjh7n@kIdau~)t^ zUQv~DI0hj{p$TxMnWS;a-BW@@Nx%L5IHOGdzjOjA($qw+i2ZQdXDzzfEO|;C+Y;a#Z zvw!t5FG4g)SZ*qx#^7!@p4_pS_h_xb`*wj~b?L8Mg9Gh{oxakY4vf{L8n(B(*=4K? z3`x^4nKOp~s*tQ55}xi5yY=a`_7v+MO?n^hNePJxMWgudvQlX96W-JDnF^g!f`HHB zKtxc?O-!WNqm#&h}5r}^lF`hCB+^ z!tK2zRc%u9oOI~YCYkgIpdFMT14CFOuz_f2{LH__K2&e41*0>!DHNFz=Ph|}Xz`J? zaG+OeAP$|@&%$}3d?`oFq=X&XVo38NS8uP1-nsS|n{#<6w+Zx{m%IC2 zY+031)cT`iEEEQb6BcBsID-eZ3WQr;NZ;AG*-oN((GRWK3TGzMDyG<2EC1^Vna5q7 z>u{T22P>qNQy+}6z&;I_=i5cI=WU1Jj^9y{phfcM)iqAyNobuS9L$(a{=X8xyI_cClgP;$5cy7F{ zJ@h_s7!@wt+00G`9|hS->rbBU$M=^yW;*)zr!)%o)D@Cusb>5>70O5F3~Xs{-3J-3 z&*nJ5e##Cv52vUNzJSYdSAf?z?^yU0R9*C`PsmfB*BAThId_WcM_a#$W|Ou+P&wDh zLqfVpcM%_IJBxtL_qe3{U{qA)BiCQ7;Ju9IA|n`E1lY}NwfN$9ZFub8q`^Zo!EbNS z3zBg?*b`D||E3S`%IR36tOG@Paz*V>oJxiHBCPDLH@(mYz7Y4vPy#NMF9wVaRacVI z(wKezgx=6kAdT7j&|e4U)K}K2t5aS~3@BGgbYsYmXbsUo(H_7RBcTUqaM`5QeaqM^ zHWli2SO1|L@Y{bA1q75Pq^Z8`OgfNnT6`rOwkS<6l(hfjum5CK*H&%h)$+ag*U{Vg zjpOlaTBR!kLLGay{)0zn6pF(SMT#X3GWXmBGTK$Fg!+SllE3qJl+VkRVC0bbw4ax z@C{~ekr~!8Wj|j=^AOL^^`^mE%Q<_S3YFXq?8~&&+R%E4{Uyw2ydidw7hKyw%HE1%~neEBR-T1e=G~r%|^=6 z@gTS!vdkcI*@`iW#@Chs?Oz|E?N_vk@)g>}wyu9TiqKpdk>r=&>rc@^g%`%=2Wc3S z%z~z2%!$;KWv&6?3hGvV%Y__qM}#jOYCmG+KGijGIruUWWo?t;)EL~gI#N52@V=Wc zAdkp^MCf{mH|OajPd^d-28kLMQ6Uv+0ehFrlsVn*d2VjQHk_lYrLka`_($*j1i9os zmNQ9#Yu|j^4)Gx`7MD8t8f~^E&B(y)dj{+##<2*oN7$ph_z@7qy9oLyeBtjg*Tx_1 z&u%{n;Hjp_8ceGVXttmAIBJ3yga#QsaHhYp$5<4g*=dG~y0xVbYGzmXH=qX2 z@@JtO28oNg$Vt8-!At`oE*r?0NH>rm1e4uR3C0xLyEk}Y%06+rFU`(hcJ(56y40pQ zz?UFAz+dd4y_3g^m%-`9suSja0NO*dQL61PJfE49e@HKn{3R4AG0g$D7(3pv(Yb@m z?_l2^L!^J}?esdk4D-I9mfLfLok9^~UjO5k{)dvR(wt`LV-GqB71|0Tj&QoEL`I7E zd{i7DaV@K^Y8XWIO*gc1lo<_wUed+NRvr%S{CRuX`9>s?auj&g`O#VNt?W~RM(H`i z%;pso**;`^rtN&cH5+f-JyEzw=8FNL-!_Tj9U?8_|Au70fqEzUTpA!Mx>FzzC@15z zw5Bk?&Mh!eCzk~!AC*|4`lxsgfxNZLH4J`Vh#1<>zt99&0Yb@^9y$SEuX#rlX7Cbf z3d(h~+Cc0*GW<%kdLOgpkH#w-=l?_4TSdjuh1;SK0)!AGxVyVUaCdiif;$AKad&rj z_uwwU-QC^Yx=;Um-}`vRdFY4kF{($cT2-~aHNPpQq5zU;o%x_Sb}y4sy|b3A1!KN? z0m>V6mxbT#p-|pOkp_fQCxDIz7!Yx^TyA`DJB1`gpPZAF6$E}vX+!h1XH@Bjv^)bfoKN25Qnso{l~$Jabs=mgsD}=JV8}mLrqlVoFG{^0L?Ywb zokC=r<)R#dp;6%L^$+V<{$Xwt4yZuR+myiyaQ_CcADjiE%x*+};y(k`x)Kc>BaXSG#6l^mbD_Z>VEqLVum^vy5Q>snf&Vd9Ls= zQ8xey1@qF-!i5;yutO*xBfgMR6bR*kSRM$%A{enAS31hz@ zxm3)#jz8n4ky-H=0G*C;R)&FnuA-e4I-*+25Tua9&EDxU_NnNupwgN}bpBzr4Z6|H zhKoYBEQ)lq`o5<+6jhlAM$RPNQUH(DZ6LkI>s;KzYg5KButdx$ZQsAy+-)07R?hNF zB($NU)YUCS(Jm{@S;&RoK_C{Ajo^%6)jsa zM8PNgU>>N;dxPJ5j7W~ypA=2x_sN}F%vr>$Spag7&pm>jMu-Jz^swr_9Q_S}f6+@P zFC(~7CSNFguvZ<7?fGQ9Iw#27y>xvLU=TOpX^AdYFh?Z7vwPzKIJ+N*Zs*2+y|OJa zKqu_)`w3#UNKGGm8$LKqT?}P9o5u~=QP=RdL9hVL2w*96?Lz%yfRQF>1#KN|SuT$U zV%I+i0_?=H9==%mUEYR)5htqIkZrf)4o%NSR^N!`@nYM*rhy8SWGw3-2I8-*?*fs) zpN^eRV956(!{!+HH_m=*oO9}ZBno8LC7nxWRUm$Q_C3TL{?8_9kd*7ACLiA;#qt|V z+&!)RzZj3N{V>yxE>3)(;^GmHe-(oU>2nVus<|(JlI|wW_*AWLbTxL{l?~z(QR&(4 zp0Zbmo(joB=Fq26{=#pNe%1u2pS^ia(*vV)l8~=mTa5hC?pBmCDOSfKBlmZI^ z#mbi@4cZ@bFu+Wo)LW;LDQ>GPeFiPx{Rb^`FtnKUv5ORX4I~8p(e!$W9H8Ni>7@M* zI1(9<&&K&?W{-l0>t^rfr!YD= z`%^%a#T3OfTNn`18?v63*^8Cg+m|@%^us3>_78v%R+8Xl#>eL9%&9qH*6Ebsd>U>{ z%3*?hC$&y{teM3-RSr4#W&b!dyF)!x=f4J<^)cex(XX?yKc9ETeY)h{l^1?;Yh#k< z00yH9T4Bp*Y3ax{1p}l{R+Jz1|DhI(v|x57eK+7l_9FJ}m5g-{5%7$?H*k~ay~^Jk zio~9fu`V#6BnB5%e}>og%$BP!=U?>~WNQ6)(M%!+Y;)(2^7uH(Nnj{GYAY8?Sw7R` zHKHFCRSzetaDS@y&pZ&wbMS=$mY?NjSv)d_WH2Qd9okrOH1e?@2T+P0pNv;r3W#K6qsDwm zIXWP?Z|Xiu!vqcZm7|@>7||N+m}PB(kb8y()%d@=p$6vi28xxqGf$YY-$NsVLZ_JD zl$X?a-vmfvHstDsKSoQrs%ebnS8#N1poa1PBH4cqDInULj7~mH7#gFxn-|V8YTPpF zN}mgur%DchGv`v#XPPTVygk!JT4!^=xc&2s#=cPqVVQ~j*kp9%Wt!$&xsP?Q(&x}j ze%;X-t|L$wViexq(;eG4dOGpg#d^>D*?lb!_E5;7- z(oDFYUc&%_n-|!yKo*;MQeLxb$VuG z#6NeJoIJHOW$1>&rh;>A|vuZ|j;nv81J#5S{X}WP*QQ zAeKxv4TDM#A2H~(6A~!FbIzQ-5AA`nmrKP4yb=ACyxVJrVZMnNc<@FrwvXet<)|C8rLndq$ijCVm^Sfwk3}8}liUkZ@sB8W z80|g-rHA$m-}d4)f>U>pHUI|nU64e)MdX7hWYj!eT_`?A=di=UyR-J6w>MqpzG;)j z+xJ&GAyCVgqrhKXc0;vMDpXMlCm}|AkC<4yJfDQgz;rM>EBz2uSPyUpd40ZZmjx{F zoLrwjUsjMK&PuJIKmO5(si-figjh5$R6a+>%{eC~5~_bYxdoT8NCLxixJieT_I!$9 zWl5TVZ4@NgEzT)sHxq?4)<23G;~~zO=!GqV9f|tVkE}&NUKr(GmcXlWih#}}e`^9H zq|%c>oY9&)1^zQv&OokwHxM3-bDbOm4tW zdh>wxi*fP=<7NS>kq3#}zuc0C#;IAhK_}&(t4v9XjZyE!DO=zs8MwJ8(4={+n#l2^ z8%`mwu`&-8axer9d2g5Q$AWjRG?}rrtZo)V&zF$ z{ryAfPhi?(ZG#x_pL`p_Ffb&CZ5@Flflzf@~7D84e!}) z?;@i#q(AYPfowOgMX|Ee?|b2Bw11>1g+B)$3!sf5MYY6-1E4jd({_8E_orI{`p3%* zKi;3Nu>LgWCmoh}9F?X))-toe(0r0SBmS!HamA`dWTH?d>V#-bQ(^&k_9t~&6$9cI z(d2KdXtxXTKZj1N71$cg8piY9#Ym`e?5es7Ke3sJF9gGJo7<>s5NBE$R)t{j5fJEJ z+|i7&FO6~zT?{|^iXCH2z68=tYyrQ{5je$rYRi+|5)Ybx>C~1#_J80a=a)iNlMec6 zj4>NNx+60VkH|WPUO$%dFuv#ng7SOVu`%;}Py22C(xg}TpmN)vOfEdiM5s?(z)0zE zZ6Az3)_*I2gZQGS4IU!U~-;CVl=6ghG+8JBYUHa%bpw&tFdG2(WNbJop9Lsau-4 zU+5?}H61DSR}S#Gb^5dd+kH;G4o_0E<{_%fc&QjId*5*b^2TfHGzm$IL)S=csH<~3Ix9Pdkx`H2X^ z9~$Kkw#LK8VpJ3MoH&l3bXaf(UkB+hmGuWXM1ItO;4x44Y*&Ru@Dq>Aq939k#ywH~ zP?1sfZJ$#}USMK(N$wKk*AI7#RL(D7x#esEOoEHqr*6V)^Qykp*4T{bFL(cb`!PXm z)LS%`3{ncDL+q|vF&f|BZ9sTQG8@L1+dvNuQPd#QsWKTe|WaQef9Z0)*I*D^j)c1^N}gtfjdym z&K6s`8XsL$lZZbaVE!A7n||hA0#2Jb13}sT3BbE|x<}RLobIHhW~1qiJ+0C$MV4SS z_2L*~qSjYctv3_<0#=jGGl@U!{be&$aoCXAB91O=(c)-pC@hmGY>FjbT8$@jxNuAD zvd$d(LW>p;8^ujBRRG3yaw}fbcMh@qb? zisNZMWHCa-`8=vTnpFp+vyuis%AKj^v06f#Q?JhohmgpH4ffOXzU0}gNTWD@A305I zyt(qVTBt$rv$&|P4%uWB%yHUpsLZex&H-diVs%W@F9yK`)N5n5DbF5In*Ll}EeK+= zRD}ET0A@mSbY`j8gSm!%&%b|56*ZyvHt~u=uF32CF}(`M>R`{YBiz?dAtC zBiO2h!d7+#+wpNt0LoGU_oxveK7_Pnsc4G=*^v=d6KiG639f8_* zwq9l5DQWeL*O&v%voOuI);IA(dRaF2-Ayp5g;oyw&v9~@Z>*Mnj1SjD6lgN zq2MU@J01IZvyuK`N-}<^)HhXB+cDD2IU_hJ`ir8@!$Ia!uudz%W1aIkA^0>zlryP7 z`n7Htf}HuXP{;|R(st!i9-DgZP?#6mPS|~_0qedBp{dADFT!4A$RmNih!GO;lf>*e zY7PCOl@4-31vFB6_<%b>DyQ!c$NxzjUC0mR&?3c8!z8#P5$)}5Gvy)Hb>rrpGMl=8mt$F=%g7Go2|5M>w5~zz z{wPJu=m%iU&cSPYH1b-`NnVj0s~`3=RehXxxt^H~h2^MnB>eDrmG2b>X(!#7G+ix^ zRAB;ZF*RIYrd1W;j?g)?nYnXKJF9d-8RP{xUaQtqS_YU9)rxAZN{vwm%%R6 zOdGYl`UmCe#A9QQr^uL)dD4cP#7pcoGFfz~U=&_A_Zv!ZjDbZ~`}r~g!Es6Jv#zn?MW~_r>Fk{$W zEjQVE6tH%=$P#&W&fsqpR|oZR#pbR(Pf1~Cd!#80Kc zxx0Gzj)UOk3xhH72?ds3hss`{8PE^fcE1mXEJVXWo4;c!gSSMd2)I@KGZNPT`|d4G z$X6-8yjSfL9z``Mas|X`UI3Z?x7^$K>zBdn==LR{V11Z4J@bc{#H(Y+WUeoV7q^!eySMEYlNQbees`) z(vAQsrFPDt52*@*MPp<>IOhge*-Cd|UP*@Tz5y!s{9wVzW5t_v;yjy+FlLWjU2GpS zvRXM(T{PKaQ6NQm578~^L@!}4vx6d8qvB%uhv5>S%pd--R$CMtYl;?=iwuSf>+Co< z8mP}HM)tN)?s|hFSKMwz2XVHj9Om9NzMCorL#va`z`7;S4-zs1E0=dFMxzYZ7K@u& zYS}0GsrE}LvrP5_{ze=n=6=qR&ki#pC|FWI%+EVVQ|L z31m4zj|s4U`{!pNpme>3K+m)AJo(|6+1w}3>2JUg`_Ys#=m^Ss9t`QRxmXr zLY8fl8@H9MywPc)&1teulzoERm_Iq6RUs(GB+<#28b$Hd*-}(k^f-zosM73P6K{{9 zyIQz9Vj;aRtaw7@;oq=BxVIvAY;uxA3)P&~g1p_lZeDV7BauZyK?wP5Qfzd7@1KZ? zr$Iscn(P^(hkZV3EvHmUin^6dTCNlVJ`*nZSaE~-d)VBF6EDqF9=JbWF&=w*j{V?f1!n0Pq)FfK0HnKV2bD8;goZ}xa{+> zSy)t#8vFLEPz*-f*0Bx3%J9nd(eVU*6xn5%u~-(Toxv`l3YSrm-H(@2JJAw>-Z5-R zlEVkGpkZ!^G&gBnI=;Cvc^L^EY-;)o0M}xL1+*wJD;(LC1d`<;ET0Qr9koawYfw6B5k}!rQmib=joe#4ovH(%bo_EaxaQD}yZUXJeVQ8U|0Z=e3kQ9! z{y5#uj%a$aF6pk?DZ6K|A=nc!md4lLG0x`x8i47?ARsTy;Kw;&GJ23b>) zOqn=p_i6kS`$_YLygNQxnXju*Ul-km6)n3v%BFSD<@Iq}I>SicvcH3AX@8Kb5X^0z zpVU1hU+i*vaMi5>ocyImSz=_BU}LmmbDux_M+Da>#hXs?J-O3?J}lzri0Bj@Np4D| zV4=~k?)RN`(Nl47v_*)lr!NbQ&eC{z(Qed2xKnt`G=#GLFKUS#E!^z())b1E$X3sdIHIJP^QBP@v%EEEPA&o`JxrDc(6M0%XoA zFw%D6ZSlb2%Kg*n0L>UVnd(x7&UHfDw6}D37xq8PjtTdeX+mFE*WjLQ>TAv{%;J$? z8pKvAf0rd7tCJ3XZHA|SLDPK{B9lrg{=J7n3xOCGl!hc-#XZ9Q;6JbuVZuC*j)Rjv zGr}kZSC9{BNM;`1eo)|`jeX`vox5h>XnAZhN|i!*Jmng0NVTB|4g0TRGJ|d`93rSS z4`C_IvGzNCyyt+-2jp8WfAHxZ4HsnbL7?gxqs>W3j3#G?wD2A_ z@qhIUjVAw*mAt%n7}(nY?DNYBmVWk!V&Ht~<}L;{<&Zv+>yrB`2`6)YnWtEabCds7 zmCeV4=XJ{FidpNe0{9Hk6{{?!;6&5;qX_dA80KLA1Dx%UD2a3W^*-ze__qE=iULWt z*3bG$0Z-2QZt$pHGEh{x!iOT0oBqBXd-xuzd~WZme{5A>e@bH;>1BPEa!#ulu@Xc& zlCBiMF&YV(R~H_;Ff4dZ{>4FT_9m)n&RLd?7ywTbiJY!GOizJ#6oyb_b@17Lvp^`o zgU9FVR(!L@OJ_1DDIg$W|k-?BB3)I!+2Xmy^`gCSLSWLlXb{$?mJuL+@v3`SZW zDc37$-@grU(_s^AX=Go@aDydAdvmmm621T+2BC;E{*CdK`BPn&>G8{TR&VasDh`3d zfWj5aunrG<-y1!_yerosCAZbG#ESNwiX^Y0xm7;s5hC_f1$q`g(&~E2nJV>=C8bn! zC9Do72BT!RxBR1`dkEiAYM|%Cfe~&=hjR|;8M3LZzC-Eh4_?aqVwL zvF-^i=4)d26n6L>rLbdY;a8k#Ql;Mgg|wPPjRx)ABy3oh&OME9BAQ7pfd-KW#Y903 zz~FUF8XJFIBZ}nYDlUxF4F#01%?>yYI$(IGaBy;$V8^|uo1brLNqKaR}T2DIYMLU zin*JjiO~9k3G4zL`}K@I^W7j-w2RFC>Y4G6l22kf3KUo@*F^^mt{~T@@_bf)usus4ZKcm{8Da9i_A#j_zdqBgCI4Rn=qXUx^fco*|vQuI;( zfi8psbkYIssUdlc@C>8nLiTgfiKWE%62AOfh62GVq04!J$rutc_D05N1KMdru6Y7-Z;%ga>|3|ja`9?V@Y zL87mDyz)~9&ZIn`g-t0;LtE=PVk8)X!6SMZXMu=b)-y5W0q%VSU>;G23m5-nvLMV}6Rvu<4p-s-fe#KXEXT72MpO zQs0U5#(pvYw=iM&Yf4axqb@k+F0OJFsL{A~I+r3Q-sP;9!sTjs>z= zw=DZ>GNVahHZ=RzQuZ|i{2D7IRsD+#wfKuB-U0q2?NPu+Fg?)BF531{c)>cpqpL^5 zLC;CP%kq_9RqLeY)y0vu z)H6)Y(BXD9b_Y3b(K_OPNkr8S=j2uWXs=6e#PHp9dBW1~a28x^KZpJRuebupdY=18 zl9PGpu~pnA^T#aXCR1+*1hu8lPB6o@4-}>lgM`}-Fp_SD-i-*Ac&rv*1h8Kqwo1kI z+!-rS#0kvy(A(G-@ zK|F9O&()rQ7?ugO$MCO$-;OzXT8G8fX6`IHLz60vP?3{68l>ljpP8(f?M170#+(Mg z{UmHk8V0k9dJZ4lU>aEO{K~3}$U9|L8yDY$en{eNTplYJIngx8A76mU666PSW~&8y zqkQ_)r#{+MBE{c#wP4Tt7{Szo%(ix+jS+ruoTK~!^~l%nk^gIwzLL~nRJ*hHy7Cuf z@v`O2Xo{5d4mQPm`}ER;Qs5&SkkbmyL$?m5J;{!g7)q`+GDiO=@~itll04jx@0?LMs!0c*aCxbYBTb7@kowSzem+{44ILit^)=M8mj5 z_Rj<%bf;h{GO#v4?-j=kH17Ev{m~5DMuqX7Y0!77cYBO?5EhLX+aFhIADDHZ^InNR z<>$Wv9WKe^^AlBp+?MkMwN@xoTooo`vNq2*5~%$uf_E zZA469RL7mlEIY+BS6=h{iHS`{@;N;pz=mOZ0&~Nr!7*4xJ&?yN4bY2X7GyI}Xv`Yj z)z52dZM#_AO|FO|2*t71`XEKnmbM&;N#fLF?mFNKRJ#ZoNvOqsYc!NLOC=}ZwLJl<3Gbs|ji;Jf znsLh~G!OnZ$s+uGQu#IRGd*=V@Sn?7H=ATGNj**Qp7gT7pxWO2n}+e#5Qn$m3m4es zZJEQF&qaiXK(n;}%%Sp3{tDe@rAWyEp|1Fysdm&kfxDl5H5*DCwm<#*kr)QO?DKbb z-_AtyDxR;*3;iPYNd1tJ)(IsIW~i zC_$xotcEO#Ut%1inMSpAE|f8shPa6f-?W@rlu-wW3~OWGc1D=QA%c_hVQ@)?WJIa1 zgPFW}#CPR69b+fvQR6Y028ZgMFwlx!$?(ian52&4$Dy0t_eT{0IcgXEb%Fi~t-1MT z)W1PD+qLf1OpvEo_D$c*k#&#a>yp%e&37!;?2YLfx(Ct!yB8~C?9u{To4<#s{h3QO zyFc@Md@x(zC$n8PH{jyN1711X;B@%39^%_$|8cm;_n9shUqqD-DUX5n%AWg8Z&#v0 z@WoN?e0yy5i-rz}YT$pN=3ADSYno60o8}kbP`L22h02GK`{tJ300BpAEVgBf5PrvM zvl8eF`>Z2+)&{C7A&I<$=h@_O&zrkq&t)|@`|D+?A$KLu(a(pqpT2rus^ejp#>qcT zk)Z|0=l##B^2>}}d<~v7!Z;yZ2Io0KVfrC4%>Lyrgb=+Rxt2RUl_ju}Ik_P;P3}f% zreX9lWh@r?hvHX=OhiOm*5Vg)Y@cr;1?+|WEaXuydm7sM9Ta)4eSig!pJg5noq|kc zLED$4WD`r}J3I$8YL0xQJVH&;cC0u^e^RNU;1rK&#&_wI=(M8=+*ipiSy;l>K`3!8 zgsRe?+_8g$FIYFk+(zFO@&1YFA7maZz+eHLv|v{?jE~evo{J5WC%OWlDTfCGiOjgW z8&{sDy?1d3uyby^1RiqywCmotLU=w7=TkX~aXy6vb_He2>btjnmZ8qcG#^w^7-C-e278|-0JD(5k{TRWKSQJ#wB^2`wF31dtH@p6DcCVubjGbms1j#-)dhAn%n~$gAaf5 zjy_su3sOnHP8dc)3kUkK4f#tb53BH3+0U{nD8B1l{)z3_^;-u`Z*_=S|IvNyTI_P$ zcD0mhA=lzO|2>)j$H9STY{QBo&qcZt2Zevv!p6wfOx&V-y0n#ctgMFyK7n1E`B zO%WHG__LRcRPfG^7;8PD-mb+|htt4C*7aUl z@ymQti+#CDtJ!L?B6(RzvL>}7%A!09_#wh0i3^MOO7k65<~K%-kIMg(R>+lT=ksc% z1RrNl4eenP(><;1w6q0I-dI_Kxq$cM9hsNAURYZZ`Pb^oQ?ceO>*56w1JAa8u&{`z}qkY%@Zyten^H@DSEy|7$lGWZL zNl(GYTi2+a?p)kkcMfR1R{1+2%P^nzbI}x@SEA@lU&Aq0@x6S$?K1fdlbU#Nllf)& zh_D1v9Zig2nOZ*^Qh0jNI6_^4>mBcvK&LCA*J6y9xWeRTev>$=TvgU$P8z6J$r9-V zL;IZG*RJto7wM;1au{}qg#B`SK6xM^+Nr5eNN1T6n#sm3kvQ32OJLG>)ZT&6w+=qy z!?Yb{OzGBeIel!noi9Wfu<#d3d_*-BYQY~$adu^~XLZVMNqBti1#N%3JqL_xtV||8 zr=q+WAS1OWa$m-#wxHUtXF5mkm#gmBfal*4T399lrafJo96aSA8*li-(3fnWwv%Il z_WA)?w0`Zm-!PIQkQMobPEJ(tkR1PvYM4cX?_-#uBzJVaKrngeg`l48W=h$pdenYf z&ZLlZtaW6KxJ_6>n#db@`=s2*2y?_R|Li68d<;>u?V@~T@F}p7%GnrZ{6IEbSX5~9 zdk8PGOOs|x3N|+6nK7$~rIw|Avd$0EurHfOPaUe{tjMn?kZ(|oAhc@v{n{rTRn2Pb z(l6(**z;BAuW%dR+igFTtB@~r2F1M^Xt={{Hsqg*Xd+5-j5Sr6yF;}WlAxVDDVATK z7+mt4;J<$H6j!}+2)1Va7Y-N;3N=S`)Z$KCc&&a(3>Lm+MeGasshgO}d^DH-K5Z4& z*>Mi*twboMs6kKm!OLYb$}>FCoUlI3yt70-I%|pws?wlw>T38QXFl=y%Zdxn-dYkk8^9{tO#JO7a?^hA|XSji`8exqiEPb zoYvq?z9U-f7YYigZ{bA&=D9dT{_OIoS<&D;?o4kfRQ&17*-WnLW{?Uz*8y7c`TBW} zTDAI{?0aq!$Ebh+5HJTa1sykl_V+n0IX6_Mdo*)i&KkF}Jx1j0>Hg{hTwCpbv40p} zgKm68zRAYgpQcD!4NKGM3PCN_biI%9ttoxE$f*32QropJcjpzgAHdWqTmmvz_D$9w zX}MYWN9OMRx31N4DQ>s+==DhN2JgN>n4p`D>fIG>*gX1qE1x*P{f^=K@|I|94rh9s zEQX?A(PEQH1-)}FvZ-u(_F*75UVPzFCGj*I9ynnHi-JU}TF$C)9b$F5v`P*`C9K=AWIBA@}>r8A}-jnV!| za@%sP6criw8=%sT1V=d_35;?R!{5VIIY#fn`F*Fk&d1Mo@WBdB+kun!p%v1FtqCW} z`L|%5h8m7h#113h!Y}0B-Gs`j z1|M8dVeRg7*7| zFnyjr1+p&l@{;gQZ8FBSX+YRp84-1E`W{)NZES zUYyv%GMdG*7KwfLk(ZnJHxENZ=S6F|ir_OAm2gv^wYzj86<0q0xkQbeZ?*?2YE8*c z_hl28>ZZ_@zGFI`V^NV1s5D4MP2}hO!7$}c$=@ja+3j5_Z9)mKMB5B2n=B|^6M?pT zllobeCFDD%0Fh@$#>xookxwLp=1z}t-p`H5!_nIAy!uP`w$Bplv1(4;eMdx4NUc2( z7NFV)H|fnq@e$ATM>g6;(xR-65G4l;U%l>N4&~Nv=@P(`SH#+fRa(MaC$V8<&CDlk z(`cba7o$8?vma#y?gRL8gO*+Qejt2wg#2m_hDDK;?1Jd`+o6XPUMZJw%-C~Gywzxz z!X!wf_M?-5hgNY9N06$7Fg_y0Dcw&Adks0TD!Z-2B@Yga89Va~@1U?}_u+*x((#AYO3wV1fy;c2O z7)M5TLodvMNekhYFcUd8PJguR^n=qpRdyoHIv8;kbd)nU%1s1%dG$L{yof8K3!Hi-PeR5s)EmM=GX#p z*4}&AdyW zyr6Gz8{TLlJSXtlyJCXa#-SN4F~;2{+2klO%w+?~AmUK6aany)l!eP!;lk~Fei19t zyeMYp6U4aW5OvEpd;|EGK!HV`VTsrzvq`G$_o@);q8BAmLA?`EYGqTzRTyteaG0oHkFZ0&*gBt zV!sD!`z=_%TWFnspV}zjJVz!rthKNYt*-)LyjnH5@;QlQY2H7~Sk}|(ewwt8xDDzn zN5o+e7$=Kj)yygRsDiD(RwK8vD<%H7c2ZUg`KBbsD|qxmYmiSy#2FTr*PwN%&WAMB zPcG;f1wyNl6*U268t;g${k(@T<4a-HZxkdYb3d<$*)AzRRE0pPCwTB7WZ|Sn-U=?TTPD6{UeSjs7g{@ z*gPb+=%0JMcd6?)9?h)`Hqw8?1Lv2i=qwaQnY z8zzTCn_lU6sU7w`Z5K1xLbTkCgSBjtjNCcHfK$j0;YxW^8?g{uRk8JiqIF9kIvTl zu?FwD^j5_3WV1ZDe5&Sp=Z`O1{(K~2;_pt!nwZX!<_aGWX{=XhKgO@~=QEwF3sb`{ zeJWAGkZnw(OqmMqY&D0HRnluJ;bnYk@mZhkzdrWgA3X$`1Kl#VUsH6F*y3T^7SpF> zGD9B%tkD;<8~x-aITclhpxi_%eP0h?-&*DelOY6+IoM(l zn2{{Yh>1zgM_mbf(<7?8$I7F5e|o;&{LIE}C=wgy`Aj!_v(e4E>%!%4+Z!?RmC*Op_UmG1&l?SdUjPFXNNrpcz-yb?}sfCU3t?pD1o>{0@ga3{27ULQ3)? zn2Aws>)<@8#<)@bzKF zrwpmCPa6R&$)(a6fyYF9N__!~_2S9luc6ZY1|S>EE6g}(y$QGSKrT@;!(C+C0vk?{ zV8TgEBvMtuq+Pz5#ORy#ocjIGG3;HHrG9T(W*MwYvA%`e0bPMV(*vHx>O9ge;-g+$ zzUDV(YfI0e!?hH@SEr5=^?exZ4mgv%2)&y+%QR-Vuz)q|wT$b3ZG=c?f1J}ZhU7iA z%kKl4rJ9mpBJG&#EP=_x7hG>9D>1IgKA7tTfF)l%H!}4Cm$g+q?iL~WCpV!d0XfxH z@1Jvs+WHU-Pged3==h_Mb&;&n{e_|ahU6;|Q8d4d4O-s?f0<=Q^5Y^BN}NEPj|HMB zE_dRXtXOebNUlmX6xqJp`1|bKE2N|v@GCR4xmbHFFI!2#ksO-77KkrF=hQ@;ALxFQ zd_`xAKweN|)Shva?KJU4&VQmEh4>MKbFo~>N!X)+ivQS+CeEELv~nr7}G8utwvHD38=Ra%P7 z337*Tvg+*VO72kGL~l$)i?8ZD;Neu_!DENi6XaXoDFII|6XA&c1J6#gT3@xe64{3J zJdIG$5nrV{+=ehi^Yq@Av7C`^-kyA*rLU4xX{MN3f=HN8y<$(@evy_8F8PgBP(W2H zC^>jMZLN$7bNOd?Mfr~Pqre{1Xzl|eY41s0$S82h88l$vIP>T{eZ6EI{(Js+TCp@5bJo^F*btG^?EOG{= z-G{7zE$J>B;)aw8^ z>1RDKt4a69PwlVvbRKiCWSdnK`(TpE2E4ChT$eSL_oIx%WwNw>hJm`ZJ*shy;acVx# zUpUA+VC(4+K-C$yIOo=MBTeC;M@R9E_ggT@xuD``0W1j_O)~ZQmYI=jfw!81$I}I$ zW{uZC@?puj0U=b`c}Xta0dEv(ROaeoyXjmFH3lJ4tJB3h5*`EKqj#WYS?!JYhUcog zd5dc4?Gf`tcC>VEly<*{*hDe)JOTf8Edl&3W*ln)e6&P&i|O51>plyNYa7FAd@X%? zC(Bi9l#pJH%%1~m8H(B{TFj2GeQ80UUGRppK^arJTb@VyV|5D)d6~k6N(|HWS)?29 ztB%?8**GUJ`dtBC`|v?IL!|t7Zh`ya`(AZI|k} z%JDVQ@YtiBep5?Wx0{Pez^!?*Q@>ia3Uyfg2;6Rj(#1y~lq%DeLph8NcT2L!|9Zf# zF(eGfpOuN`CS-vbY?|pP!TNtaZ61dI_h@4ia+b%QK0FM&Je@^(#Qs|TUF>L9+>@%Q za(c)!gPdkA8-J9VYw&R2yw}2zezGJT9CmuPiX}bR0V;JKqNFNX{j%NaST$ zel+{&{@9;(`J8U}94}wpoT&|&@hJSt0D*1h9$F5Rlh!W0gONqCHa&f)ImS(-_b6WQy@T>Y2 zW@ys35vE@hd#6Z?sm#Jt@I=x-#BUgm4tH7OHc!-q|7IKbka8nGSe>xi``-qBbkC)KzD!D=DhOIb_5R%-1G$m56!N!3n5aa!%(6%T{jI-izb+C_Iz#5|wfM zXMB%)Uzn}$f8<{*(Zc2~RKiZ6*UVOO0>o>#+(+nglzR_SnDO3S7=O*d*TD;x{O{P= zw4h8Aw$F{bPPc96v$BE{1QseeB_1J6E2m*Q=S}{u({`BR8Q@`9nE(8i25hu`RYBrc z|8rVJa5V3>?W@ve`u>}cV*CSSd?V2KEHM9Jk^zvtk7eZq&BUPC&YAM4K5D~%%u2zw zi#>4l=)EIaxoQNubd(1u_d%0Qkn#z)$<1BHStg^Fse>)qz8%9)8HdgNsin!(VZ-&R zW584W7fe~ZCD>@P{@WgOUld$8rSk1=TKHXU5vPDZ{&cQMNbq4QCL#7b3gxU##XQLy z>@7r99Tlg_Dr5GeR@VH7jpGURn~QR8=~L;;eKAPNcd_^5aRm8P8u#&Rfzan|*C{R- z#g-NjD}aPoyEU=)Tjf}{;BOm0J+696EXy!(caL!kZw~VhmrNKke7JXEM2+G*JYWA~ zrH=^jiHL*t0;rJ?+h9}|Vz=2Xo?8cKo{)?Ajhd#T? z{R0ldXEYyfYRXr0PvCf3C_RBdw=z(WM_J?HY*jt_^xESXKfuy0^j%o{wQ?TNjqhu} zsU!?A%r~V)61#W$>zik7mu9N4-;>nG9tV8ni59?odNCQ$cL4EB3Ycx07l35qA5fva z;&2up@Hz`w?H+SC*Oc~R(yHi!<*_-r`lvBjSuSz$8`&HmOpYH-6kTkb%Q)q=F4dQS z8cv;NP|{tr9($Ur`2*SR3Md@D;+KPpsjQp8swbN$&A-Br_mgD|dysr$AcI_w!8BTi zd2Difv-}{s$4c@B!y|L-T;p!@ec#2gM<<-*IzGHTg5~eIP6f&8#tGY*?zP7y#X{Q% z3n-;k_c?XoAK5>TwRxYOLvwv4&6BE-HS%fv-#&rO}EE=N!o>%IysAE;qi$Ew0=HF#P$vCK`R?AWq~`v42Ap4Vri?SL5)0s(o4XXYJ=SM{@ULzRRq!?6>L&urIg5Cu zKnqX?4}(W%l{Iws9sA<%@*kYe+58VS`qdvf*X1RYugu^oX1NkL&)_Tpcbg)|J)u^6 z#qu{Sdn03&C5m&PJY}~`EBwE)Lt?ry)KlG4UzV*^i$uS6-s#wN2=Pd_ye9umqg^nj z@v6BJC_6m>F%p36O=?nkZCu)c=cB;=D3qF0S*#5uv2Qax99c=jMxl9^(o}6awTXk9 z{D&7cFyhifMM6QOCE{Nd^5?>Z83ITf7h)@Sn2?BGd`63drDU+U~ISST{%LBTbf%?Ut zMgvCXln-#%l&&c=3|(uD^__?6c1&!uyNwxVP%EF6r=a6?@bvHq9MKX0F+rSHs!m*% zD#D8gE}xX|fXQb7BFa$is#S6?z_HHMlS{`N$K}yeu@1dxOJE1f&ra`b81iogbbGJ12xl0{^BNz4! z77maluSgnPUs|5D7Xf%qmEb>U&J<{W-gOeK&3R?gxRg9>XOYM?=&wYmoTb(S%o^)C zbX+?d&?yM?{IywYD#9n{JX3qGJ`6fw2~8i~w)~j%Mcs<<9V9)`Dh;p=foum9(|_lf z6j=5JbrHN7=uoBOPI)TmB9z1)op=@faCqj=X7?jr%N$Q;9y7`~$iyTws)OxgGYAZd znD5aW3%Lwnz!fJb#q3oOyeCKWr6=a9UB6fXFGZQcF%;N%cMEDk8EV+QV5-%@k?-+| z1X>N5>_hJN#W*cvnIu`YA`tJ58jo9&+sz^_p(y)1FBP`f>DDJg+Ri4}%fnlPmbS@n zd({p^w#^au#B8o`tOQyJ9z}#Ly!F@-@qYZRRl3x(kLV+Hriy_(U&B(fl)f6}>g*`% zYb53K&>c$1INjMQ@hMcgw0$@1bV{B&Si?lY{f6066Tsl*`I#$xJpcbadzeu7uUMdI zJ{?pA;ww$(i2s@pq{~ zS9qxSh3V-LGBtzAtvh{;00F&z{^vlN;z){8(&bny32Wx2XGUr<_FKw3aTTDphs?j8`3Qcy~INCBm!ySuv^q&p>~bA}u`X675;_nf

    EqW7;x|UAUqDtR=9@wnMzBb zc_#2^OG@}@a{CU9yM!4w3J#%WbN_5m*9W#j<={yNB}^yuHn1{!ns95aWP6SAY@=ss zO7(`+c~hJS2hFmvF^@8ej37>*H_c`-2&{8=*e(GeK44(08+UTV?3gN!jL{taZr=+NbRtifr~j-5dV) zd)ByruVCQWY~Fc45DXkJOD?}&NJq*tOdNk?{j{@2cy)H7e`C(gsqY~85F(lP%uT^d zJA;v~-Vz>DGJ7Ta+a`Bn=c{M47F^gcui_GxrB8??27RyRIlb`1nZ!E8^shg1eNMWh zHis@{W5V5tFO>GICz-n%4?{5)4FZZoi%Oke#dmJI<6)UI7u_c;TST*!JJKS&@!7v8 za_SZLYR7OGg|NORd>&R+r3++diHSrvwaAU(o0T_z6~;nevqhPkquav(MK`?AuZTb` zwp_JzTv!fkbVB6mN)ervOqb>;x&}%%`dNE_bENkM6>V(x^g#jJ3Qw) zM&e7hd8gO@y;;{tiwX{7vRU0+gn^89yOtL%AmjS$Kq+vYMuD?MwEA+0Op@+$L*) zBZ>A!mO9{dP}|d%D0MTm}Sk^c~18U31T|O8yB_c;2fTZe9pN$T9{uNA|w5A${sG&Z32Kz;V{B;o7oaL8E2Uf63?)+EMSbLctkDn*MyQ+ zu=Li2Ik&$+%j;kw`lPkTk19dd6#ASVqTk7c>Yx{p-iX=D!HTJkFe{G5P}3lve*Yp( zi(3W8M?M@vzk4eBrRHe_1|;|-(+Y)tuFVg5;2p_9{RsNFcY7^Ldy>72PvKd4l>K1g zP1&Gv6LSjtt+ z%qVL%n#hUgwBtmCHfUI#`*y7H26WMQ_yAeSQ29?+bk4_N%Q)7cFr0#Eu#v63b(){p zEL?4d@XL8Am_4LW$A%}If*r187n51e|$df@jy*>qg@jo4ascADla{MpP>yg!MS!}&PeRQcr@=O4! z_KSQf!tR?FE||*OobNVCdlYc!&$kDEQi^kkCbPN2hdyRDK4V15kQt4c-5QqCbtA}p z67*^tX5j@+9#&Cf+=C}z**bFq3FRIR3L_>?7eUb*pGo_vJnckJidK|azsF;#_H&Zp zQIE=e{wwD8%bSQ_#L^-N>!)f0^H>=X!QXb{B@&Wfqe8jExuXhvEU4`vm{r121F!~a62CEB#s*`!SD`Bow7`^KOl?kG2L?ThDh>U?%;`{VnLw7wI@y%4ziFptUMBUK z-+|e>xww-^e12(9>34yYYW945~lNTa69qZ%}G$gQ6Rqe?Q{4Kx9CSMA2$^#^BxM>WyT+-$h>ec0+=UT>6S zVZAJz(^2auskbAxmw%o3{GEHjr$Hfttn?Je>(b%Uc)TB)8h0U|PJ@2V9iUXvBV)}@O6;dd^q9!v#As)GCV~USUL<>aa(}ZYktGNm9WVOz?zP4A{Lppz zm!(pA6SGiWvpQ=y<2|Rr0qt=Yrg`ohlVqQ7=8JGI6jyAlbN8~KaBvO@)LsiSRNT+G zMPOd=oYSnjTZrY&&o+6Zs_e0eq%igQyuQ~go0)GBDx%8KWzIGEA^n@FkkrI6nTpLI ztv1HMbn=M!7uYO4mxrV8{a$ayR>ZG>pZ0nDik6EZ+qQwJ zhyA_xB#YL$&(zHo#f5eVnCe7uqztMTKT<&d1B6yu2g%6h zW6#&DUspSF)?ayiWul>7Vzd0W?Rz5f@}3egPHnSOr~a}uzOEs*rmR;!ASTF7&=H|u zWNkmEXq2SBvylGRJiK*BZ7Fe#bG1rhP1rS|`b!y%c1}DSxo8I*f$9L)8zM;!aA!an z?e%@VFg$w|&YWH9;Q}0OeaU`vx>noxX_}mIt=Gp+FRoSb1sjTU5$L&FBm)Mu)SkmH zeInXV^WW>fZNOcoY-zdJpmWaCmQ~K(mmgjC{SErN3jb5M2pe1i_AAFLy?US*09Xew zqV;OFa*wimf1*icKl}dBvhs#@5a^?L;;PW-YI8=%k#HFKX6o~iD_JPwm%tZ}hJ{<4 zEv|nxB{`@F0%|$|!UrH@@9_>;GR#W`%L8mn293-+d!x2Wo*H^NBK(ihAS=19x&621 z)7A=MW+YTvExJ?q$FBldxpAbT;rAl}^}qH}@Ut-=b^Nrf#`pfnMb+=1P4K(`DqmeV zW9!9yr`zwaL@VLH&1%(xzyWeq#<5htpS}s$a1U=0mM9?nYMK^VV}s8C{Mx<{RZq7LQlM0uV?sf zH=R#)-xBV+qy+@-H!fWfBrqQe1oNW_^9?g|Fs(?dU{DApk(_zMCr*d!7uD+o$GaV6 zY)hd#F8F1U4qxlD;seXAZY|iocZbvPYV_VhUIppWqx((Jk7q`VJ&zG9jc>lrDqtc! zTSylDG9Nkj8G@U@c}9N;aCJ{-Hp1npai8EIGj3Mbdog8y@4Ir0qal&MhR&6==6Rhc zVqlZL?z*VMT4{4Zp(aqG?^_mk$b#r`&!P9sOl^jo5O9yvIt#aWCRzAy`|3%mV-dGhXD&Q~%rikXN0gT;pY&QnuP^q(!g(_=OXu+nxi%7$kI< zM4?PlJn=iNZ%4fB?x(KZZuXmJC&HO4k8U_WCbs_(aVX!jgD7Y7&kCS>+S4hiq&CGb7GDRQ4!S= z2sy0Kx~3lEBf{vf=G++{+wWQrAF?aysiLl3WJP7TR-5$N&zlB3>Eygj=?23|(UO57 z1TSr)W=Y%~1T(%m<}+ad=c|?#x}gnzM;?A(4<43)rO%HW%R>Y67NhEDoXXkFwJfRf zw1-TLt!j3cyBV5^&mv8ptuv781bEI#i>r`??FJ9c zfzBMwJ4|KhTLtR4w8`U5Y6u>TZD@EVM||6*n!Dqp~JizX)T*E0d`i`F&4+l8*Q@ zUONJ*P_fOQjf38G`T;~vF<-vF$G4!_X)z~Ia?v^>FnQ)cbmefO3;jryNAR=7X(l=+ z-+Er$pF=iezPM?KL&v=4zbNn**C_wsyS4QZ#Y(dLc}ao1xePY6zwGx)Ch47`ceN`n2P0aI-y1x5GSyBCIz z(*y2Lkn^{t;YEaH=s_cp>ZGWjhOONDLClN3tU=FS1uML@SDz-tN)d$NzQnhCcfg~p z*j@FmMzitP%lX=p=k#x*cNOF&*zdKnU%nrSXloV9OTdKbvx1|24NSQ)CuFcv^kMDy zn-4bz_f_qiMwz2VVxeDuI|z>w^T^}5Gyed$cVmC)-7*yqx&N6i)};4!YBX1$o3uWP_}H1c!9U(LtU$(Gk9oW^ssjvB5ipXkY*(jFSW}Y^e1@1hlchg+xt? zt5 zQEyS&@l+?LQ`%t!c63GM@X{n1!7C;+R$_DNIAxQcOVT9jFSgksz^)(F$F=d+8-X%= z7hC{{$htcOSA^_@=uznVMG4#2`3Ow1&#yKm_<)TBCe9vuJrW&A1*Mv+u4h8JNnpoI zDB@o3aGZ6NmQ5?w-=EGy&y0q0r$}t~QpGG)1m)HM?RUQ+?2n+M#Ezu3u=4qN4;owK z`{81Skx9GJj!KGtI4>~qNfq@F_ z^k_ai+6gu?1Mal@K>e^W45^N+zC7JMECj*NXE=^a%wF{BM^w$F_=!OsM}}CFN4*Vq z^b_gs`+p{!ef_2-*Re~(XD9oL@%7-WhCfGLjw`m^DLuMQXS_iEEx+AEbHEw9yP%yk z0e-eIoAyl@S7i=ezGkIW8+=yzu4M#n^Y)0ZCecBl*cx#Uj{5x43PO%nji-wh$7w}& z3R1Vr-L(;A%4p4GwJS8U)>TivN>ayDj4{y4yE$y@1OD9f`i$DhYmFP**Vo8aHDF8Q zWvYJVxm;&c5BN&Kpe^7Yo17Pi|22%-1#7&8Yt!uwQRY_$;sB<8aWI8CEajtMjQ?IL zG|~2(54}rZ#<8efw2%no##wk0yJMN9H>>*c7z00HSC z;$`E8OYB}MMSOi>o!8u;%MO1vo1k4>zYZM)Sg)`G5G$o}5G=3=xgKyIY1v|X1_>&k zqM!K`yR_>8;TE{HpQ!t=JtB*r9=2S13lci8hWa9Bji8y1Q#+hOykJIH)%Xn8xx{Xlpj|aNB#j$=A%nq7V;|l8P*Yk0nXS@w zx2q%a%BPu3S!kMnA1J-*2T_o4YrFv58CCeF#B;|Ci@i5LVMiW7Cu261=R1Yyfapf) zP_7mft7K8%zt1;QLzo_C{K%xf7YNd;MzD=0zpshM%s1pi=w81kOr`gzD}dLSI(44n z?nM~@ZFmdSx|%GjG}$yq$$S&|93rV*8WqQ*I2Np6Ke?zifuiP3pUskB3MQ$|(`xKe z_GBlr#w<}-E{%?og9~N%#@rq$B?(9N%r;>>^OWx_m=K_gJA{L!Dg5Z6TIP|~Bf%Ot z^(Wc)LL;_99dp0%0UU)JKGU>r7g`+T?8)SGsI+V;75~XBq?7jgdud+K8zi_X#jI{h zh7zfZM_RF4uRTyMcneauO*q_lQu(WuC(5(62c7*Yy*^S&g4)LY!k#pRIdy1UdYDb~ zn;fOlb+T1V`V_HEOqOzIiz86jZMTNXU&zYUB2qhxpbT}WNnM_XqWVpiW=e65KU$H9 z3%9UcigAc93e)U%p_0K%iCks$fnCb-U%3`{?UDXMZz79*@Z`gMnT%Kx0}J(q?cx@x zUVpJqxOJFA+z6OBHqWvIw!alN>p25 zuKBvOsu%2cIY4Uw%$M=wHOC97*<7@buOg-;!5Ve|w7SXd&etQQZmhcncYok-JU-J; zp+TLibcNJ4bmSf*^GXhO zk6DtB|D2*W6i)s!Ei&KRe1C}p0*i$h4aYW%&$5JCfy;f|Ft5TexSnv}gL zgJ({+Yx;fFxvgmpxxcq#{BDcjn64oHXiY1KhbH_$d#3(%|87Zwz1Jg@*k)-xSVR>B z6fU9mzV8{Bch8yz{unKcZXA7Y{{F2&cH~CITv+T*7rU!`)xZ!+?)&oLuWW;QdM2K8 zsbnYkB5Q1cU&RI`jp?a=?)eg}mj#W!s;NDWCHI^P))kz>WeprF)h@-i9*+iXQUDmG z0RwIF!{7Gkg1(IdEM>m5+G*~-2*1bAf01Wls33o+TG(+{viLw&6F^z1Z_uLo9ziE1 zbv1dXAg)WaC?7#~NAaN}OjI%a$(Fhen~8K!o{ zyTFXyZLfsm-#EEMNhiQK8m<}3K?5@{`&R2Q(o6X%)TL20X8(E#j+ti`WMOY|Nz0;W zs?Nd73>vq}pNQoHM;nJ4Zg^|AzSrLpv%nWfe z=q7jP@*XLM%iRL!o|qf<*7u_+Zv>XJRfh+OA6hgYvVG|cYVoZha{NMEHQO(U+1~C6 zk!d1@18x)fZQpoG^KKr(U*)-f*2QspbBjzkaV(eHv>={9^ux zIw<=sE&UxKBsC{^jc&cv0JkpJwiwkAZd5*h9cSkBlH|iH3jK%_!?(1;ehcF09aLH@ z-Z#7WZKvfa{BN05p?;yS-<7dasqmJ*D=*y}obJKx#)Zg^;l5yy2$-sd=bMajHpU*n z6ZN#ks{HnOzJHP6s~~>Z!e0Q}$|0#o8-)la!K}AI7ILGqBRJdBqazO>%@S3P^{P|~ zH%=s4LHk?dA^TWEU|OUz4pN7(Kh32)F5KILR!q8k$316HL?XUO_Wj}Pl9{hy$z=PB z+!12-eQW)B{3tz>E{4bK1+(2r`MX}+4OY6=u#_Z72{EadhML6hN0;~~_1t`GhocWO zh%Y{b0$X&QsVZ~w#rYwG#QMz$)42~cK4-x|cCOQfH7R?E?=@Dd0szm8=Nz88@ut~u z1qbbevonx!aQQ>g&1%Zx(H*ZDqW&|YjaCca@h7pR>QDbUzo@TR6P`UT@)6EY{C4#` z6Y4EZ9ws9v?F7{?R9*;szARVW#`tw;l_i zeX<9{Go7ysZ)1pzzV8cP>=0Wi@%QH0)U#lf+lxI^Zj(>KcTU;OJNmxZqdda3@j9}}N;)`O2ZtSMX z0sjI-d}B81!n%cJF8>*o2vj{8?cvOrjVJAZz@8@)uC;c2eQ?=kYI{`E4X=vHF=yF5 zUsFe1c-l{cbh9CO%sAI+vK)6&j(h{p2Mt&Dm51~y76e<>aPUkSXmEar0*kgU{UxH7 zmdqGg-nvc4YpzGK2ED|li|&@shMoD|(Sk)sVjzH{Y!U}Ut-F!O+?q^ils!E7*O&V0FU`KkMh zllb+kKxeR`{{2u)y(^LD3G;m6fq@O%54envZ%z_6-^%9SU7YB70R+;pcD{dReThF4 zf?qs{>GA8d*by*ZRx$Hr73d3IT>%bt>nwtF^GDNuIa8O%J>-`T*TQvZxEE~s5GG$> zeuTOo3lL?-z_?N70^JuUXS?xoc(kl`Cwjs_!ZJ)@-vsk|n99ug&e-t<*4| z;y2SjEJX%#q|npRw@MnADBM2pt{k!M7hFZH^npw@N@*XSlHC<|$M#(O>dvN+^eZvh z&8Go#$z7dh3gaS1qI!pngyE?h;^x>FxZLWp5SBN6$i(xg+@x1bn9gw)`vPnu0V-aQ zKAXgNy@xR&)=h^Lr;>Cdy(U>xbQKq-3`&vy}t%8r2q- zu|_Sw75)#}iucCO%GSc8MR;^S%74Vtk!eCL$ymhp1my>{N2%nfu7`i_RCuA|ihahP zmm{%@7fr47WDd%Dx{!<$rJ#L~#rV-38TgCzH5;+_3Rn)6kH z@MMX&d;N?mVrUkmBg{p2M%cqBFIRBAjZ_fLd$#W7Zy;2G=dy@WnZo!4{r1-R$?Ghn z4q9z)%N#^|s$gBQ|Qomz}t2}*-_ z-=B-)1pQU*Gx?vUzC)aw(XnlL*iewY_QuMmla~a2Rg?7ygCf{XFEyT{%zMfnu;9de z^ru(a>PnZ;GF3Z2?GqZ8%D z=>EiE-7oTup7$vYE439UWy;yZ%7lC;(EhUSR3Bjx;Jo@$LKk6X+sk4&=TtnK74wB~ z%V@@Wn(kQ?!{p6G0l^5Q8>^Bq|6m-nZOx z7T{69y$rnijZ=ac!6<+N2amsO zw>XoodA5BBYDrb|wmCl{LxO-Iwth0c5H+fO z??!VJOu*noeDT}4JHQ0!OAJm>Sos$UT+tx4%WjdZe44*Hio#;5tzn^Y$v2Kp&9xoZ z9_F3PdH&znkH<}rXLG<5OvUsj$zqn_Z?i?4&Ox>axbrN4BIWo@^|~7=J>7yoT5Z{g z7?yP5x*XN3y=R)U{D1_BT4qEYEb1g%(Ro=v;?+*mWf<_pZD}S^(zH_seu=KV;|QNk zDEx35S@W#aLPU7{^TYOp2lbV1Xt}C(S_NDay|`#Pj^jPL41Izu#!wP7kf#fc^UN2d2qbaoE=G~+yvqF?uHJ&@S^~I z5`9wLIpc%#I5kK}rF|Xw8{%VcGt~pT^-c$kvnPxTQjdngaSt)!f3z$M^pV1!bW#d{ms;ABmBC~WY=5|QBH{@DatJl|(|(%% zl~|_-EY3r?R{iJ(&55KKJ7!D2g259X033k-RcN}ma?#CFvwm)fknJE^!64?V*4Vv5 zHRRUWx8LN9P~f-qt3j7~QkA*tyP!1cD+wI%IM7k z&NOuqibt3s;K}K&a<`y^Q6#vq)C^yh9s%PSU|VSA;x&47Mzt`a$CzSmFQliCqVb%1 zY?A7HH%?OQ6^>U-;~CzKNf4EMcHzzBF+d+tkMc9nbvsZf)Bn+NPgEk+O(k(eXmiJOmPi9hA65e&ci8QjL^LE|(q30=Q%M{m z1Q$(|RWk=L0fyMN{( zF1&H8!dw&nu-5B@DY*nVj(;nw?xP(hTrLcqo2~BDAH!WHzmuHNw)#O15|U=Uq_{%N z#A?Zjx{vpml!pzZd_ZRM3BbEoAhh2&4FkKpFI$I3>$y(fr8ksAkPJ|rS@nuz=78T< z3Piq%+GVz@FG6?%u{iNmeZq1i&mx*-`G;Iv{tB@3tIYD*W4Ry+ExasCZZn;_z&^1h zo4>*BRh=&SH(icktCN2BmjvY1?mf)BiEWeoK>(YWqi6_Gr+2-aMrk9~MBO_x75RJP zvX8OG@4oA98=I~!LHXa~O!^MMoPj*0r~-=A+Tx9KJM_5X|GmhTC##M2DORMw+o54NX<39XxLti$c0Q=7ycg5>@^_ZW0SL{NUgty(5;Y3O#HB#VMaA_tDnf;;Bmwkp) zkO?7U9IHj}Jl%qhs5s~Bbp7VI!WyE9UiqzF4|=IgS+R$=X+f@SZ)%L^UEwoP<)nFq zv&X0Sy=5{F9AuC58gt*P8}mrmVKd&<90F7gNjn}PmfyaG+FNCZ=BaDIP5=2OxQZ9J zk>YOm&s+rcoWYG}<1O~d+X%{dA;*zw@>bE8aOX6TA{|GFIge`a^SbZHX6a-7ker~8aq>baHOLFsVI4CXC5xv)gcTV5xzI_X^FtvAFEI)J5kuo zpI&#N4G$rJ>1*u!NFt(ZCG-(*G?{2JsTXt9njOXr4XhvTKm5)oo6V4u{DFg;xqJ;R zU6Nt8{NF49v0j_Ihz9&%-Tb?ui9a5hJ~e99>n|E3PE_f_ey2JCeE)8C!B8?D|Iu!M z_?zUv{hT^CWF7fG$H{5+y}OQqE>_DfGG3cFz*#Z|b4>v7MtY<$*=|h4MEww9%p&|3 ze%7U6^$0g0v5G;>_>%Om6tH1aeV`Rtw+1c#_C5w>wGV4tw7q$ReKW6P+$a0f-CVQS zs&@5b=FID|yOA{B*Nn%EX|<3!?v4XiEhGE3eFM){`d}g_5o-Z^=CQl%Z27R_a3jyg z)t92R%ReudGfl4D^uSo9kZbs+T|DirCMXld?e0){8FUyv`CL1FS)^)n2?B8+7pPW1 zSeblwr7)gEW76I!zA-RkiQ_DSaCJ|Al3USSfdQk#A8l4u)T*!HKAp?J=KQyxN@258 zb;0C+o`Yj&gWG~ie5T+sT_`Gr>Eutk@=oKhW;Sgu_0R!5;+N&$lg0#1HTz$@7ljen zS|tAvywpkP{3h}lo`Nh~a&>B&oo1vXETE;Vx(YGuLX=aDl^7i^k7mXC1&n?EuWRKS zvq-9*kohYqyX`8stoE!GZFC#w9JNl@8aPK}Zshutog}6gaam2Wg4njh%-=f6;QSV# z`Bci6lPnj}M1F(m=|7qe7t+TXHS79B2d}JP6{QGc$jN+koDlGxuVFn=x-^BnNDkTs z^1foJj;k^FNl6sCwoLN++I-mr<5L)pP~xAK>fo^5wPYPvp!jDC&Vc5Xmhnl|jzlJu zEX2*yRbM9@-WIiw%l6=~-t`=V-<&IvkyNG5E48M-ht~ zt+}DS&SJ*;#i8>ihNX4R=AVW*Wwe;;20TJyO9sC7dQu>3nkIH<=qb?yAZxZZj&O7R zzo9BQ$K3qF38APrgcMZ-P_8x|7Az35>W6I@AB z1mkU5@7^<7-z&j$idAi>(!ikSb2}z&jB#DhOB^#a!!G9NAHpBdA80dA=peie5Wx}q z@L=1a*5%_^MR4@!NG|)UVpo`}=q?N&5*bHk4xRXI;?&h;>T&PEm4KwA9YAfcY~vOL zBpD8>g8h%P=Y0`S6*jdBbC35(#fu+L=kG?>gnjTFy~_6!yPLksj%xb%|3c(m{#Zu- z6d(+78`hm^T?Azc&BiRpuPf@Fk>N`fA?Jtc1?78RC8r&E>V*7ftlx|-E&A&H37I~x zDBH}8zTS8~lw!<|dB->oeUZe@gD!yg-^QRi?Ryv_V?MmCWQ$nK1*?c3RT9&^-32_>~IAjg7 zd{Y-u{~n$l13#c77G`~#SLaBuD;$VM62LdWMp|o7$#Mu&%XJ2h#S=N$SWOhAQj28M z7u8WLIxSt1bnZz7`@>{;Y#Jqzno3}G!Nr-4AJ`>r4Tp(2QaQ@&9gMiE-OvtAjnhcVSQmg<7p+&Uk8o{i(L>JEMq1EmpQ zCr0#ZH2LZ4nSl|w)_1YB@#nKv=#GT)U0el05I2&91Y|_X+>?m{^kf^rf;(t}`Se1|OC0ez1 zd%m-mYUb0CejIJ`N6Q82lUsO%d(&H=<(6fsBW%McDsc`h@DAqxhx-5a^$IXpMb+gC zVv2FUkLGYOtF@z+p>icV3Kmw!p?*WoXZkjdeuuoF{<)Ll#4I^)*y*$Qr#T9c4-C(k zN?Sau;a^6?xx=-FSJ2>XzDT)MJRZN__6YOXE$BRWF9rY~s_Fqebt5eAeE`=wb$U^4 z9o`JVS3y{N=9RvlvTT`YFkzFE;+04~F#_?rI_(>35z_;7%&!?{3eL}2}spny56`BeUmUJ@eBIf0#E&il|w{yzPW6*p6An|J& zrMN}oCGUkM1-*Wv)z3~B?O&^1?NjjU3~GI zJ?NgTIs2H3$oRFI^E`6sg*fV!SzY54uOcomR4F3XEFO%DrUH>`;D1}|)eu)QC4|Y$ z^E5T6Wiqf%?^NETyOAy0HBU@y>cZU2iUI_DzN29tWBKvbyBK2XF9I8ym*HND3^YXa zB05W;0=WybnW`T)6KGlU^dpbmFxXx%2F;aeKqk|8C}Ifl1cM%Q-A^Uv4@Emv{Iw?I z*R3e}Ncle98SQ`D!cBe^7zzbXWXXh$@@(wws|Wmkv%@M?RD7}V07$Ob)z(vGsL6ZL3w*?5{)y!jO@f!d~xGt>>1h1NkBl+`c9@ojmUi!I?uUE(r&wn;}CJ&!9 zNJG_r=TZH9JGNJ@;n&!K@H^wmz8|+8B^Bou_T3F9RS~$C8m;!*T^hycK6||0)Wl(Y zz}}sv{G36y{#5UX_~9{&1YONaP<@`Zbp94R&q2;Q+53Q$VCgl;h)@m5Z7z1d`Mi=J zBIOp8iL2$22AWOQeDU1$5La>?WTF7EgOiCWuUVuCgdly!?jwQT5_`PLz6wOqYeNi@ z`U+U>9+%j1VR~sj`gXdeSmX`GWNkFBt5d%hx`g&wQwO8;;X?q&EL--Eej9 zL=OTCi|E4PGF-Ip9`R`+c=Zl1ox-vPt8x+T>ZfxDL&!n0Fi;7h-DeGoY&!6M0u;ky z47?h(`_#beFpNH}S1w*)@!Dg8EY& z^;AXQf7)}lN?6^T{9EK;4zj7~4oCwHB7_=}sCj+xIx3iy7%JP<;NSi}Zh$#jcT*Ie zSkQCE=i%-W@Q0wGxHF@6cs@1bpTz!1LDZ&ae6b$GX}gg`S+9KxtJYh{f5)I|7z(_Q zvf&7Ut4>YCS5g_Kbyu!i$ehQjmxLS|y<5JW=f8xNr@*GF^=87#Abp z{abSu6^rrCCrv`$yLR_I7+vL0HwqwyGe+mx#}S3kq|-TpAgBiX>-}^LBIH5pc=B16 z(DrZ3X|sN=XXLh?VV_2l4D}EZn>l-opW|&-&%1)dtSm!5*Aw&%<}T|0f*`PAAh%hV z*Et~GS3Tpv7#HVTQ@FSIR1+fm3UGulUCB00c{3HYk@6ld!X=@VxU=d;$F|M=DE|9<-UPSAoZ92jC&8Sq~V`&Y_0@lWPtmj)In z9_YKv0s9>7dfoC!qn03%gSOY# zDgWO??lkG3bm`yg*S zA5(wXzxmOd9!y&*E6#m9P5Co1&V~+a;f^1cx`HJv-jQ+k@nhRFu||YdQz`B-kAc0z zT3fnVZ9?l`Zxd+=;CbN+XnnIZ6Dsk&WMm%ExqYOCYYUh4b2i(TZz0oQnnP-9=#MxR zAI6(EaL+a7nW0Em^}iG~U@yFEc{gBcj&34yH1|2i=*2EJnnfc74}9srQb1#7^i$el%J>1$hMk?) zwB&!PZ$e_b@xLy|_=4*V#IDSqmEZ_70M>U3Z9x(fUrq!!+{{|{z6;v@GR{(9)_1++ z-tnDkM~H90L2)4O9WJZaq8zXmS=VhVg_vX;j%o3WUE&q~hs?dMM~NL?rRYO>i0FwoY<+z(|6v^?@V`9h+YcM_SWh-343!?72gUd@&6p2Bg~ zUt8`Ad!((9?qXz8GYqdf;)AR2oc-p3XFY#32>WoQUlQDspOzK;&26v}UT-3*EcQ-O ze;_U}c^@dqxRcODP zv{z?VdJ?*ypr%Sj-K0;SzHx-k9ANB)@mj37Rds9PP0i%nf4#E!Y4XhOm+aYA-3g!c z)h~R|UH=|42+9nB2pms_R}WDcfEdEaGt-`ocPLsdmK>$`OA$ApGCbT9wS9HNnoq4D z6>Kb7G6yef(KCz@F%%I}*l+dsP4BanBq&n;B`c)_Ue~BU(q3YLbGzb^A%!idXLkRB~;LDO-+WI?-y2G2gGTuy_*p)fIJ1p&S~bCpjL zJaSJT8f*4jeH=ZIxE1CKqMDmjox*CEIC3}Ws(N~CQ>@GTTSR$H5$*g2;=3Kx1^*KZ zpZTsKer34ac>7GJz@^#TYc+Su^)g_X*B6t!Lpk$v(eW};i(27>E6pu$F~i1aitZ1P z3y^4D<+aiwfc*w8t0bVNc3N{u$N_kW^n^e5(2+coZ(T;iG7V2=`etb}t>P{vtU7fVyW&@WU9GmnpACn{h}Gr7rl>-nA)_v)S5hfJ-kG;Y zOIdLK;%%JICNs3HyqZB{FNqr z9lRAj%CvPi(`4D~8157|lhnusYZnX;;8#rXX}O$)Fgqd8Ep(T>wRJ=IfiRQ}ix~E; zA2oeBQuK+cTdAx1QnAwPqdJhoX#9rguwS&&n=^X4H|fDwiryRJPxcr?)7uuldD z*L0i5i4nQxAw_1%q7I6J+ta0B8Egmz_QWJ0yl%2{Zsx(hr>%-@Xh~QD+2!5U{_Z`oyta2y!_VxdchzWc%KO51YL0Xr++_7y?A`J zI>o7X)h8Bk0T9d<R8Az&v~ z_S?ve1PppN8%wFrukj)Q!;%;_)C_WT7D%^vTCmB<*oGBBc9!zj z;$^qRw+^|o8!xJ0fF#AmS%m$-{T1N9nb%uw^Mq6VcOcG3+V$&{v{s}f?2#Z=Y2e&2 z?emyk#ev)^r6dgyJRA;5_4Ct3N+y*r@G(w`F*46CZX4-0V2b&>U;jS2-e4m8X0`;% zlTnwG4id+8TU5XO!B)+JfCAxcepqKdW9#BRV{v$1#T|?KSIXXHt&><~b*sc4`GQF* zTz_*fbojCm$u6>CgIvzDbfuVG}d<_h3HoY!XstU-aRy5XqH&d0Jl zv>f(&_U|2m~P8lNRxV<-5pk{rKn<*2Xgo#HvZ^z)8$#%0%3 z0wewCojuo6K268Iea7oZ-JJ_@(hV2%GqY@^Dg`!|CT$+Nd~WnJvQ6fW2+V3#O*7s= zR8W37MR{4MnGEH3?ris+!jRBB?YOLhR(-(E6{? zy^{Ekf|HN<%0!WE>-W$~HcG6~Swg9ev~TTDB4hg+p$5<`WI{?BrUCgR-E2PaI=SF| z$JFhSu^q}Vo`x7|;YR%3#;-$RuV>yUer-vn&qjFyXV2b(e3dHok8J!mGiI8xu89Tu zQw8CYfi$Ka4L;wz&ICcOUYmg!AG_R!+=mvEMj4Y$waq04Fi5VZ1JQ^sETyE7C*H6M zi^3`MAdN;i^c#1!2*J*}%gjAB=eAbgG2q?CSRJ}mR;l(M+l&_PZToejvD`^|95%6Z zq!W7bc!85t*`w1<@_K6_J-s7;w?&^Vm23JqV@LI0#zE>n=*a(^in|skxH}YUad#_PAW&S2Q{3IPxVuvd6qn%c76{4C`~Cm7PxjG19l0i1GuO;| z?sW?{jB$Ua28&R`>ZR<7s_AH+_zweD|b$)YW$H!pP>ur2x6SV870Pe+Nbk1x3x$go2(WczRR19GcKI3 z7b+S`CfW*aPXNxyy`Wquwf}f7I7c6#Ga1yM|Cvh9{@Ez;yjI{unVoDg7EMJ$;pY#7 zld^FVGbj8yS+LHFVA;ZH%qKvBIs`hO2R&XzHj_}oZMj~hxIFx|*ot~P^LhauqxO?-iK zA2aW%i&xH<@x>OWFNIytU9UfY|CV9XX{|08soz?&}Wk+LngwAct{k<>o| z5XXkV9a|?YOy)@GJEA(Z^H6%g$Z%JO&uAN(&*&7>IN?KlQ1x9H>og$lbiCgU^C7mM z`sw151@;+Rm31YjVr83T znSuHd#99vfgqOXpKzeN}*S$$K8&E3&szHHguBDCd55BmEwQDX+YxgIZF=-!?{T8{v zTb_r&-IH2(*x{tO5&Tx$K^t1Yy~IjqcAZ(8jB{+cj|9@0g)vNndMgR%9 zU1%^zc}zTn1^u#HVJwDJ#K9a8y|yJYx2^z0vNp)V*`_ zbgMSxP`7zH-MeAwFRTW4k+p;0iH-)m{@>7+&HX>~vkUjrURaH7Prvpf9Pr?X2ZDuW zMJ_sCpWTn+oGwtVqb-i!rs8O^f_@=yS?2N6a)A|ra-#AXU~>=Kn1@-7RK-B=-~&%V zbkgH*DH$baBfT;?(a_zS6tV9QDXd->3#avyWYl^?v6^c|s_xnp39*Ljnh6+{q^rmq z+e@x1!#hplbcY97TRKoE;0a|P1-OkOs;)0o+CHu|Rq=}4%dq3xgo`U8ZBB8p;K6cE z-DE3Nm(287F&n5^x9OVh!0gg}UfSsNV%_VvyR@;2pvp9NGkfzIQ5U%im3{qq55qS^6;CDuG8mnHC?;vmn0P`rW0)=;`XN2y(9Uw8wmk znmVrWNciB-2{<^R&gbfVbX(CCx@I*)>UCy+Lu0gI%U)+p1{{Ffd4RU~-7fVFV%Xri z7H>h`=fJ_CcNKg_gZn|O&0W0Q;~@S#AH!l^wG{D6M*UFluHBsdL+JkONouys&G<9h z=AVdTbz1E|*i)=7H0(42FTKH%3NJpxa#c?qOWHhzw1(3*ZnQp&;Fi6wLZr- zaAA(XNB5DAH|&Oh!$1EE^g~Br9&j2pYuEhYMy(-NyZNZ@3&{c183U-g{Hnh8XW*3$ z_jl4x6%nRLf~n69Oe}4q46-WSC}i73`BKx+SOtz`2(&VEMoVB%0UvqeL7ywJ?vDW- zB4WQu4OB>fQSvjmzxP>hvJN)l5^LOvwl`I0`@+zCP1PnfDI!h&dCoB0E$!Wg?|uZY zhEIM!m?<`E>l*r)miOA82e)kV_wYARx4BK%l|lDb%?Wq(1Bj) zs^$;$YH25@A~EI1ruS_{>FPHpn5*Umq3j_-iW2c&&Yugobz=VhJ|e)JWDzXferxoV z0IztkX)rIVrLP~wa_mssUgYj%8XoN!GWvKp5Q|L~IsM7^-NgXQcHGY1xTY^psAAwz zfwASOZE^X3nq?mvpRk2;QPKgVLZD-&`WH@zve8E%3*P&sVY5coR$pM?2Y91*dkTY# zM4r{DJ^vk0XD)Ct?xBea{1o9_u{Q|^SoPqmx48K{tG@vyY^vrfFcc0~1(8Ej@%~q= zO>Sho`&@VXrt;9*xQYjlvA3>kL2rRj*u>gh^abuT{--6QQm3@1zDD~tghn1z_c!$q zV1d`4b-Ufok3rrg&U3Zb0$)94n?Dl!E;P*e!OAeJ)`2R!v;XP13J3FkUq3oN0z5BV zjX-skx0z9AZx%G>xJYfqox`hV;kUQnvAc2z0m+Klh%`frD|lc$;ZNSNzEl*NoKB*9 z{^GHMwQLgAj_^E91u1&3d43p~Sj%REr-6W1RtOyzJk6kZzjn2rzZn4TFq~;SsET&A zD2g)p7Rd-+R0y9GXFA-gKsSbUJgBJn6gC@_Ey4P*UroWkNv3uTwB!Z#{zuvW59;&g z<5CHd6{H&8Fgac$xIO`Dg_i)efu8BROsJ#vQ*e6uun5%R-NyYPKgr0pbc8X-rSIkU ziSEbsU8B;ouhDaAiIAx+un4fs|7cXa*e$?&rO_b9Ynd9Ia*lnQZ0|u)B`vHRdR$SK z#{oOof=Gd%^8C9*AC4@1P-r@5$ z?I_mh!(0Zo*l*#k$PRxWm!$zU&DHR1Ud8A@VPB-iawLK)Va{~q*z7K(Fb3uz0aYBrGW^k%lhbL-O%rT( z_7^qu3*d}Z94d7G06VFJM%+~OK3DBq0IyI~#UGLV-F;*&V4f~rwGb%mDIkFPQf(IZ z7m|S4d&})?^fC_Wtp+9?EV=795xxj9{NJ%os(fP~b9d3t(Kh&?-=RFyz>x#GkByWo zfMhLKOD9>Xml!|+6W325@%|G$RDnUmFA5*}11T9fm*^#*QA4S7 ztUQ}X=!>hnL}PY;=+v(h1&HPEJMZX(di3fU4IC6sx_UxgSpFf`l5(q4Ejs;g*L7f2 z&rO+S8efr+2E9T@Y;xOhu)?NtcGgxc%$zB+h z?TFUdL+n^Gg}k|vOO6reNs1!Ki(e6V)pm$b;788pJG2aXjTcjf*mDGvGP=dQOFW9T z`NO|v)@k`SPSTKOUJ5CO9`Ga$&=9{#G@0jM?hYSh3?@vMnLC7zY(1Us69cc)7B78j z!Zva6!}f%}zi#B+4G!dpOOV z6WIOMHe9I#k9wm z%tSgs>+8}n>29bRI^OAav#-(L6H@w>g8Oi#q^p&i|J^n}-IQQ@i|M1*2MCtw<(aCD z`f=+1yVcS${OhI5u4>1%j`o{`uj3Aj<1!-$4Nr^;x{TSQV{UI9C7o4Rw@GgL2jwT{ zJ_>s&kri_-t9ib%orjv@ey33S=|t(D-kkD^*JccRiw1C zy+@yzql5h}1)lA-we&e}_0;&oPyqVkl}6O$_uy@dOgbF*m^17X!koMpyA4}+x-!!z_CZ~+VA;jBi*xaS~h>H#Gm)_leVb>pK#A9d`-YRMe%ZyuTsilJ@;Q= zQs9lPSFt(emNUnqvov5!ubiqm5zk&gdezxaJ(t{-5xFA|EJ|heR{A)sH zxhU8*!NpjK>oSFyuiIau8qpre5i2c^iN@Q z+1o^Fa3VE3Y{vxRae1;m=^G2oP0n;}(Uk5f;YneRqTJQtZujes^R;Z zng)ZNb)%){%6l>xvi~;RShX7e<>=7*n`{>o<2sz%6a^1ec^NNV##?99FFdY_ z+i0dEd)BCfNtUPyd&tNNL5Qg7I0;S^j*26{*e7F%)>hM;&$V1mUV1Jov`qqFFmGjN z&(^bldiJlLl_xqmmD*)$i+T=iKI5V-Z+`4gyHu#b(oPY6<12t4wfqS1%^3631Wp5(!wJk*t z`H!NV<~ZjFdcwa;m>%qzc_C&*MT=*cWz$DQWWm-y3=7p%XqQD`_dY%j#slxf&Jajr zf)iAK*}g%{Cf;~*0KwY#IR|j7h2fvQx3|xS(Ubag1G~1#S5kwJ+Zin0QZ>2G2B&H^ z6uq)G3#;1$)#aU+oqD%y<-Q#{E9(Iw|N)B|o#y8@-VP z?7h>T2!RO7?+*oE1I)$?+s&R+qW&>7iz3nOOr}$45un(^n6ZEJux_rPsC<}}_|d!L zlcj)Yfj}5w0m!fh62d<4+MnX#HnWX2JOWZ7iOgI(s?BC#7UFYS429XNhb8&2iE(Rx zK6hLhd6)9yqH8F?o^;+uaY>NKY1B0gpLAIr_C;fCm$1io!uLdOWOq;hLT^r>R`m7B zrA~1xN9Vb-nVAceUcsYdI#Vbm1R5@$dFEJTQhV3^ zv6aLwI;yqUXp;%_*KfahxQf<;_zBj-B+fo$Nl=SQHB-Mnje^37jpIKG<@bLS#8fF{ zqG{(*c`Q=(Fo(X3RiRthUNC_k!-iQ4SxE)utDptY>gT*lSj`A^`zA@p!&WFEPVB>1 zp%0)y6$^|<&(6px5hef9wA6?&`y&<>E*4hEm=o3sKbDY@FL9>|Mm+SB8V$~CitaZ{ zv}lAbAxWuc2oUzHT0ma4=hXJ*c6CyZ22-1bY?LV~g$d8ak<6m4Zf(EKIM6zC18ZgK z?R;%F%0P$e7e2|q#czpj$K|yW9f;llS~yGy(go;_5ne;$~M>$>DcK3MQbJ<8_W2(Pb@UIf@`3;tmL zv0f->C*@#E8DE6W%{d{!GsrhxZYc}a4icmVele@RagcxSdiIi3fkQzd_V`zospjk9}=-ye*?mvRp!D8fkj0pD!3 zhs-freN^;Z@Q{ri^?4?LPZL?Ge>^dW$NeHYG_oRRLcdzj>&&+8`NUzM`>}Rz2KRyn zL}^O#QEix#766%xcdc9Y2%gQMZ#gZJVHU!piGEg+013xa&i|O|+`O;7TIq!-vsrHQ zcZtU$nYPBc3?f~@LdP?Si%j5=>2Vi^mJsWJR$roCdL{T(5gOjg5ItdKshyRJMOr*S zIR0pwBR)%^7wNtNMl~&E_@NaeK!{p{9I5tSXLka}jRyHx=PSKsHmqmqg%@k@t%G#hS)fc+~HN^9 zJi~=`Rh=1Ku3jbo&`Qh=>=Vcnin+KlcOP@IQAhh27MQJt} z*Ihuq=46w{FDCl^nu7d|Su4JG=9S$}Zf@|fO3E7u=8xwSrI#%cvSfllVTvSnzE@R*~-L=5vw z;$Hw^fP4N9kRE6w_p&R6djBtYP`*IDRryYjEth;|#7*u2=68@?ZJJ#Eyk!rOykG7s z@8l^-lu{1l-{rgnjQ=sdyFW@9oEwaA!#LEt?{Z;{n|apHzty*SdS2?GGJbhp3Zr5R zj2**D(UrNtSI)2H7P6vdb;^SD~9m`D|iB>vnGABG)pBrUu%_+8a8 z+pbp$1t851quBM33=^?^yoCpMq?#QlV(2cyVu&Fp6(fKYfGdJKclK6ZN{550h!)+chflv`UKm6Jd%q7%c&KK~@g& z?eN;1pZGQ#LO#&*DnFmYRWycj;x4FrB*`a6K!O(OO}$ zO6Uqjj%Y6%m|%AL{PkkvlTJ&F6ioMK_8!G;&+ZSDoBW4(w~*snJF&>FmtDl>j&IW- zl0IrdU}>&sL!v2(Ap$-PY~8bQ7aN~v;%}77FFyqQ{IT|`LTvE6FT=^%+@6k9TwH@Q zS9s|`WYAr~JR-b8DhVrBjPacL6n?e1uB$LkjNU#6!LKx;;fi?STMtER1DHNqZDp+@ z-OhTJG)1X>s0soZ7`|?J;6p@JaKATlhTPC{0pr;U5LG_*$H+piXz$xfsC+D*awQZ+EL?`=Mux~L)LiSn zckwBN1!=(4P;4~`Q(M4Z)_`jxi4nWJiM<(+jn(6ak9Lqu_6rHiWyd?rhJ zE4B`{Z>HI_nvFkf6fe6TL4U>-$tharE{GhzSdtmKX_>R7W(3c-{0Vk5m#VjK*muSe zdYk~!(nr8gqZq9@T5?yuK%ScHR&tG{ISzu{OmUkuUh@u6#CmKG?ebpwjIIqHibuS~ zm_h+u1z$y}2Wo{7#VvXYWlaM9F5Gr0*bp*YJ$P;Z)uZrjf96r!Hj_g7G3Hdw7y0Av znheX9&7hIaakUH04n!-U679-3zSQ#&5Av-0-pxRlfr6<)FdtaW_E2?q6=MCqeF$5z zZ^x2>Q3(=8mB917NfSwzBjvFDpiUN!F1j}lsEl2>I?am?9EK@D%^gJEtx~&uDB4_Y z@BAp*gNaYCkxp!$wD$vUe1nTZiP&v#IGRGrXE41iM;1a>djk7ONN+4tH^rm251BwX z?K5<2CzqM&c5mb8PwxzNi`EaRp5pH;$dk2G==0Y;tVFl-OtTw zGI%_+Gg4&?p15*>J-C>iG#`WheE|>#K6(SyJ5}1(qysvmER@8{G6sS;;`QMt%Ji4qa9cSxXf6-!i-AL$sS<7np&X|1o=h_yN3?vMtl;FLYojP zsHvf6GvPxaUX7FP6sa|;`wTPr39P)Pej$v@SaGks=JZ5Q<0v0Zdb_*@dB-sKgtlkT zC_DZpa3VznTz-!nxk!9=q*^NWS+;oBJ0^i`z(RbVR#-IM`?K9Q2ZSL}+~J)usejX? z^SD?a6WFl?B&z`;$iH*EP5b9gP4JJ>o5~Yazb&D#0NK#{O2dU5Aq2 zoCADRE!l3|H6=-hB4PqrqTg^lNQ7<*?DHz4b#KLWlhbHTjyo&C-yNK5G)=HxhW7xfV{ zR8Vwhu;$c{Fhii(JV|(*Q5pQPgY`K9VDa7up)|;# z5&2IqA7I8G}|ua9S@e0F_veX1m0)0y~)i|@S&dgTcYd+u|_ZFLL!S5 zgT9V}p2>~Nq1ht3lOxRSP6bwaRJ$_w*b^jq)qYnE^_P8So+`vff&wZ-8$b#h-aPAt z4CPEA$zu1u2gjJev&$-Ip5t3yT0rHsIzLoscND16@(0%*w^Ti8L@CH2Q@Ar0^nUt} zFqXMIn%w-|a1l{y{1LL_0-kAbE>a-kUxhF9>V8Lc`C1YK#Po6>sRo`dn2zoX2lZdL z;x@NGR%4EStsR*o$GRzbkNC!gWTFnC>}+kdjX_CQj3EB)&p{cq__y4#nB25wXLLas zPPP*y{17Ijfe$@>2uHn%2$AGGqg-4xs|~*LzOXR*8`dTjExL^f+rX2|F+dz8S_w|hEJSz>kU=!Sl2{;7u^ql-Q>{#hTvi3 zRNIa?;klm#sUpQJQE6{w^)Yqg6jd2(9P~>h1mVwCM&ZoZFTC=C^`KBbCt1M5FH-99 zs)GGqA{j2{J=w|^Vc~%-*y~orsDKMrFSuhV{O~K3;_PNf~* z6Ss}37^4jWZJ?Q(%Ji>kgUHtH@=Lv*8^XJn^8bB!m#Uv4Gsz0!+WJBi4rsj z5D3PF_9=T7pR~V{VxDuJGcmXR3Pv0lqD{-2V)TXQbfCqQ#C~u6{Yp7XM=yP{aH}c( zgFR^|K}ud%dVATKaK1}sR02BC$0J~YpuKkC`cdE@n99e|Loi&KmQg*oe9?WTB{>kh z?mAVChyU5sCwk=#P52~>yyA!7n~=W!XuR3@&4z;yl<(l-;+RWj-Z3>tRjMBX+0gR~ zNr^OQw__a)WY}UQeW(}R}c&s%vhU=%b z3bdWq3gDfRk70;A@2)W=P{e)tk1n-2xM3PzJ+LeZ$2-fP&CrS?JxJZ2I_7n(EpWkN zcjPDLn)mP1IJ^$dJxY2Ho85Ow-a@JTE;_1zEgrNVxN9!Srte(B*O`>wbCjhlm=egN z*9t*Tc zZ`x~SoUcbKBYhb;J{^U3d_Nr;mKhBPw}aJ5#oS@>m|$1t?Go@d8YEitzg_@KpqjR( z*v^V{&WBfGz{`#+Ea36$1G9h5#-VgN9ESv zePeltW2X0AUNEu~duEiC_b``NOwKX`6DT<0BBNnC%PU&mrdK;6gC@)|V=4NH>t%X?BnH@kfZoIO4nx|HTX9nR_0l!1x6owvcKxvUsV+y}!Zlvud%A#) z@zKz5wc&98_TCTKn1NZ{T8yTdjFL}$$cHJ@*=^9pCrwEQs#TJQ3z70qX5J~yv}i-} zOgEfxU|Rfe7MQhsPAog#bvzDKr-Xb zM*!^jP2ifkr^#vED9C-fv zBaZ$wJW9ta#njvqi9w0Myg(kG-ipM{~E!sg|ci48ok(DLZdG;drTm^)(>t2TDY{N&xh=N?>-{&IYtq&Hclp5n8fG~Km_R;Gf- zI#M?{K7gS^7Wk_f#U#{w`v{X!$*?I3ukhb~3CH>RZp9oY35JvsgL=#YdAe}QdQoBB zhtD%fLjz;v9ojw!$kV{If||9Kuu0BNwzJjWaN2c(lmAhk=RX6`jL0R%B$osztY1}M z@o&bH%%``Wcl<(8Ue>HU5eppcMV7R=M7q@r$gaU`3`%Y8cYDd|p}w~VRlYtV4^c^W z%LW4vk2*&lkEnXR-~NN>+Dg(z1~LUfykifugnTW)NlQe1qM(ljK$d4h7%N!BV>ORK z>B{mMzIbVJ;`}yy?b@Gh;B#;NX@q)N^9L7c1+AXS!Qxdb7X2d}6iB=PF1nTTnmOed zLp?+%Em{DtSAbxUF>$bbsnOl)W2PP?~p}hh71}W4dI@h^M6A{artJ z4J1R=dF`z%B!lI@Y7wos+0K`e{(#%;LG<`KHRqsz`sJ^=?A$U6y66Po-j$4tB4%dw z@})=`A50)u#51himBoiK(#Y))0e5(ua_kTk2+L#A+A^^~^ehqN6Fln!y8aLxz)Y?% z;p(4_RU;lVDP`NiN`Y47U-9cSlB5iK+#b|@M3%YF)1rt(ZlqKG8E&Gd`e4bnB={?y zw;>9Tsyj-D(5~%>SoVs-7Yn_UDdZ3UV(4q&#ST6=;QTqt34aq4%_q#eB!-Wwx1qW2 zO-bjsgOzu$(JPTshWYmHz7&J5@tqMUgl=(couPkyAn=cz6*s2PE0LWKB(y5@`dtRV zXInzQ%g1ebJPNM`%}2mu@^KazsMiL0|Meg83QO*xDl;7}hktDS9ya3AHqVlnpxtyXL8_^bv?rt~pjOPuK`Qr>|P)vUqd@#O8OUol-Jk)%4x zTh67e7Ixp=NWTyxn^Nr6GdI1)4@cvmScf>l%D%wy04D(&dz{}H#{$HmOcH}7FjlA@ zWU1xF>=)qSTr|3ml0K`&adg5D-*`)qlT8pvG(yweq2E-En^=(4fDD@cA&I6IBtQ0l zZc?#DX`jcK(o+Ur z9l=`XpDSwAVx9NhdHHI&sK5zFLix_mW!32)0vLL0T)Uk*L3T@Y>r2zZmtj+`#us-eIg~`(j1^{5cQiU z;DDpz_-%heZXV6Lc<|2iq_&P;icWx)(6S^z;2NN56u7FHw4jDiD0NfEB&e-{@YPO; z!*S{AjR+RiG>$o)N;nCgW8U8olLqZ#)KAeW2vP4*hyXnVsd*Y)%HR?2lj4H z*JFe?x?Oxlcjhvxzx9*$@6ta_Gp7LdRmZt{{qIW!)_eTKE|{W3lw2FwCPz-;P6tiZJ2W!XT}V}@uI9R@5bZ)-N$1B{ua${N9|{_syI~BqE5(Wa zq}#(>>K2zZ4q)x_mB9vVXt;K^^F|3%ZN%J__Uo%4r`h2-=VGtwP7O#RWcHl}!%d~K zZ-zq^GTHiHW$9-!pSPfoz?;OYhWG##sAF<=n?Ja&m)8HWb1Rrcza*C};CpTrv*UUr zucM#4o^3z=X3Sx3gbj4WOg8eJl=DVOTs{-c=jE1Y#2dM2s)?BQPLV{`JxJ!pP=W}% zB2S*E6snJZB}3py{V*_6njZN1HD9pCeXn`AM{RlJ{3tN)qp#4F z_1EnV)KTYYTfP{yq?USb7m<;UmX9f^Cw^6Ku)NRTSu9l=slgu4*%bbR^~yWQ{piImWecLdv?uHSgj(fAwMc3rW?ZP1YXRq5SrJZ7MR8R+O+C$t#B zUGWgWi2`rc-+9PK-4y6bF0hpz@xHU#C2>xZp8gz4s1bK4w08nl`g$v00Z9kGAJ;xX>f{G+ycrw7Lt`!?*7|J5p8f|aMLv{Y6X8R!WrWA980Xn? zgHZ^*n&TDQ#ed-QRjEKPS}a2%H%apvVg&sr02YuGl-7Q+5l=`e({|EW@<}tva)4*X zLadA8K`(HhMP+K8{@Fh|9iKn%WgYe&&RT7<%u>hAV|z7j{yfvR1-m+^54aQ$&|v$* zN1QgOwjhJnN}~KdQY^#)wpX<(W~+FvL59@!Q>v=E`I=gVMCpH5aL1=G`N~(f#?TH6 zjJvI7#jf3`4$?uP!4MhW|>1S?sH@ ztQ1c3M3UOGEO@W0edn5ANRWOlYiV2Poayr z31pCCnZcDNhakONnExE-xow4o`2n{AcsGYhwDM*0KI?9WU=oN6Zw*Gx3d2S0xv(mq zfeR}~OT8Q+6l?oiz|XIGyLH;BF!Mq>MbaP@;CSc19M{u#>!W!nEzN~>Y>=an;gja;UN5` z85%4U(eZ^26ja;L2_7Dp&boKCLBws&=-|^o>n^`0ge{v`v;9+n6}hPQ?c({Hj3tI| zByO5=!fcQi8MIE-%J1Lai=(x_3DTJZ+TV;yV(;g$D#9_O;)EgS=WMez~mogz9tR_L>Xr$YFH z*Hl-Lw_+2c+iSqT>i2AwC!zXPlub3Fi=&y}DNHLu#l+5(SP z>6m8~_@fpA6e+~MYQK|D&7=CX=V2w+k@Qg}pKaZ7%O9DMvWf~}u~a}?_czIIqj8k| z=mMPRtiTRDgb_e2@Iyo$aO3ojh3aL8hyzP%6eAVF%|0Ov9e8!P9Ch28EHK`QAUX1g zZM2&&tv7`$Fo@T$(qC{U^7bh29;t;Xa+iAq_;#rV3~W@Jeg%CR1*Sq{4ExiO=!5;F zfZF$fZRtf=VU49zs2C!g5xuolRQM{XvFtzfMv9zvt&p6`7v&P_%xf+4ss73qKE_qa zygr_${z0>SOwRtq8mUA?gWxoUx<5lxHrtnu{Grs0XGSK46|SQfp5mVRh{=W=hQ#jd+y#aUMxZ^G5EM{X(~mmZX=9Cb^%!x zGYq@s(>dPjq_u;TAHK2|sBn+sGpM0H)*9e*bX`d$?!Ot{n_=zX4w#^}a=~n=z1WDD z{L0~xl2aM$;k;;Hv~S<=V4ub-+GbAN~818Umz}3&4QDlim1a`^Ij;< zIY71e8Ocn5$>FG^-|?csQAu7*N8EyIz`%52U#X+vP}+w(BD)65hVD7@Th~{q_WMSC zMO+y~Yh$X=D6MASWX~s7TibhVUZXii3wZZ(Gy2cfx*4#@|84ghMLV}@h`aW6`8+)_ zav5yAUl>2RG_#26VENabDlu900i!6S4Vt_o9ZUOVg4MI(Sf)tkRra+`W?I=UAzXpk z2wB6s(UQeT7Vt0Ri7q!;R(cf30lC(hBpYzw(YU(ewQIN=ki zjLG5L8+089?8n`V_BwYxdtBM35ykU81VdKCuBHq|fWl@f|3G#5l`uxeWVN6!ElZS_ zS6Qg;F7F}(Mr=Vjh6A#6`#(g*({gu%X z30cK8&UlEXi?U20_xU`uU-yHwvY`}~+#)1YiEWf3N*z~zV~8rNUqeKchWtO}K8YRK zrHotFVtiYdT+98nW*}u%HnqRi>^-F1m)x*j(TwL(a zyt{~U$(a?0@q0lcRz-DZU!Jd?_lstyzu2Su711aDIT+uPWfMcKk1iUGDDA%(WO~vh z{r)$8c;8lp=$#^=<|~iJ=^<;8m^BM=+n#FxnzUUH3xvO1yp(Vx?>=nR*4AESZXs^C z1aaur`zM7SXa5Nje;jwU+J%lbQqre0W$>)TZMvRbRkQ$81~?+E!_`9m2h#!M{Kxh9 z$Os&5mw2}!kOSumy%QQcRwvFhbP+7QqM0wqV10LbcPi0*k^lnPwda6Rx_R`Ft&c+9 ziLtjf=}Z{I_w(NM2meinR;_ND?}>UbWz|5V(4mX@)%WX|H9RBF`~Z> zT=#Q$8U#S3e@eEkH3R1K=*mu8M9<64Qrmg{%sE9&ldQikkz-K^l~Nj!0huPtU@t{~ zdq0)|nW_FdG*>+n^Rj@+5y0~@XknF>WVS`f#?iDeHm)CCMMM8OxU~rYZFnrS?)x3% zIOf2g8W6kp^np`j()b^_HSt2Pw4oWLZzJFO&n(P2L9Yg@JLeA{pyPsm^FKbml`v(= z6xv5Iv`C)w2t_u2p9CaI??XHiL=>SE$M*jY&+!)ZO8Gh(m6V=$6JdqBG+rjDdJ`F> z!3#lTppj+w*9erkth>XMeu<%h|DIqph35Bp;RKCI`{}+yyaXs=nIJ~4w?!f1$lKW) zhs8)Hzrk-fRCY~mzj~m1xw#TrIi|;sht>J-SAhfqcLliY=E6R5Ws@LM<2%*}Nqp636Ufft;FrM1$8P{M zlR&P1mU5xJPgOH!0KcY(#L>YPvFjsDFCX+h~|QYbvDRJnrUU^!r#_bN~_jLA>9Ud0uMPJg=iGIa}$AK@~-q z3k@bf8C)1$nCx_8e(VWB8Te3L+nFhoaJ(J}o*gQ5R^8>i{Yc5H(^HJ7GaG-$c~o{y zjIhFKdWWP&w;m1sbJL5m8@xOEeW1C5cQDK3#leiw%k9dTYinQV`$P5BaGgKTWI7%BFUxucgVGMLUTH4I9Hd5$fj zln-54EDw{qTxG{T9!TC73+e|H5$bN5$ms(yz!G51fmCV~CiYFV!*1gNZj9EG4 z%ETEH+#F<^*7d%5kF;`TwZ2VEGm9_bLUM}dQKCOjEZ}|Ic60lC8PUll*OcCEq?>3q zJDd>8x7kd`Bc{=#LAAqkVVvBP11T z3ZA<@&fo3tz2XuF;|%5i22hXufnct+iOR*QL1o~*p7dx>KskQeZcqKUSJ8FXK}`2s zR2+N@rOC89RBZ7liA)_c3J1MV+Z*9oGxYrod#oyIpF1>_l;+(v1bG*tTxtgOf!lDw z)xl=ud1!^VzwFd|s*HR#nnQD?J@Q7L1vsK;0@s!TUdDuzSE*k-PU`JM{oZZRw=#Tp z@GLk6oQx(f!rC8FJ{4ho#EkQlxIJp9y}CN6$z+-v{!D=_`-;wDz~8nR15!dh=2E`e z@uoEk|BHI?G>L0&8_OSbw`Ppll4^dkYw6$@Xuz+dRwg`S+a7g~e0Skxb)i^NHAQBy2YD zp$U`&&y?t!z9C1?8VxjtJqv`^1)Ni#=kh#NK*6}Fp`HwTLzo$ zNV>`8&#)d>))U|gEI+K=ZTMGVc*sAkpTMGjrnBGhc7a?xho%cUI}C;M zGJ?MJeLr5n=kToi3Qqa5pUqt_--yFj%v8V0q*a%TqI3EU5$p^_*>J+kv3VHO|ITLl zI`#f&_sif1Q9aSy>Tyt#75eqt5ke9!6cAEWx~2iHT4-=;7he9KWbn1t?MH9=T$F>F z{hFtzu7l`(Ig^2@lx#J%j|1bZZyBXKuo#Dt``i{3QUJ3uvVa@^1ju1uC$W5MW`oE{ z&Lt!O!w{OLo8)shf_(6yYmc7u{T2-R$eloH^TpJypA?0?mhbZ+|D9glV3okjAL~Tu zVj5?df6I2HzPIgQH=XyEhq@1VGEnSnwF!lf1lOu>h^jT?b__Fq)Ado_?>$%D6&Z=9 z>H%#)hY&^QrP;b+uXiu<@uZLZc#2{NFw83HL!o)@*Oyj>$WuhZtuKXPfBC=u2U26{ zz-J+|LlXXdvkLdHj~meZPbI*13F*{fIFG&AZvpVhPNV$ejRn)Q$;mhSF*Sd>&6qU6 zgGF#Ob?4oZdD1Ch`V%o^sT{i_`#F_mLrw;HK4eZsQ(lcnTrBeJQ~%N4Xl+-Nbpz{j zV(ZSX>y=5T(leI<5A!+%*v(zt?ZSVOoQyc?r1HkVfBJNbV)L;eK3p0F%U%FAey}`T z$7TPyEcOQ4Lv@Cn0fbi9V~B_tq)~jKi17d29ES~iiieSG-mUf4bXddGrNQJ|t3>~* z4Jd!Hf2qykuxOYsPK?^2#U*_YCdT}k=j*M+dBJo%Bon1AHw*%7Zn4=*w|ru{Ws7$7aD70;AWot zqt03{kIT=~=A*o5ahA@VUOMQvJ76j#hYGwKfCf=mxb6Gn)B$+wxAfwq9UKPb^?r4s zL>9GwF;S0Mg)}g>gsXn>QchFPTqZim`|gzCEmI+zq=o0W2YPH$+id+fGxF#esNQI%xTFb~cBz~;gRRZ@8!gZC8O#~@;Xkd#fWx z`9+c5y7dYn==ljHnPGqYx<6``#q#5wEzwn0g157M!|}znbf_1gr1wssfNd^{j(8>o z;kPakw$riYuog-$cFeQnq1higI)p@&*M})3Mh>ge7-@T?BK**+-3bCI$>k8ehsNfZ zK79~f6&;UCoEUK6sywy^ODTqs7N(xJ>1af2+=@^7%xzZYM+bg8Il!8#9CQoxeB>-F zaF~cRltNzcZ3ADV0jQ`?>j=p!T#v=!-R<}raP>B;m4$NNU zABOTFta-E&Wev*N$(*bBf|#}i&S3ofFia9w$FBZ4G~bR^$Ic(E$-=1it3pyj9sm)d zywzHgkeo7}s9f4VC zn^l~Vkr%5RJ>33B@Xq-?nrp{_mri8JG3ay%J1+{kLlD;yBO)Gv?O5=2=_*V_OnEs9 zuU=0i*&89y?lq8>fW%^6nV}l+y>AkcaE;C%xoze&RJi#neNfew=auT@W%q2V3v%3r zBi!|BqegD7jkELNjKj5$|9pw7! z!y+XwHNG=t7toI!7nzfOgWqiI~7{X6+$6oE-~|NBHi)nXv>w* zD1^|K*P<f-gtKforv9z>duhn_h|&e6{ria79EcG*kAor!?Pgc` zXcmuR&2+>o@=Bji+tOu0-1XbS?>5{~*B*c>8X!q|deeq<_Kn)4MyT_RF%1GwAc$mNmEk4}%VeIrqvFPl~1; z-}R0xzsJWeknqcla97Y*Eq&G}%^Er7zqto_uN=QJNFGk2!+|5?7HX0GxbzUxGc=qvjrK3i?p z;FEVIzcUqf#cMW`G#!OaWO*j&0>wN<*+q*E_mik>BA20Q1 zvZaUP9d2aAzEv!>n;*bT9Hh)Xfm1pemf!6zyI|7&Mok@&$Ut02({Y_CW^cM`bSD0W zmxFEOy>mzBHc`LJ>Q1e{(AURxSokM2+v#QQVgvY6cySWL-6*D(n#)Ig_P$LUlhNOQ zZOg3lF7;yV+>hB|u>l{gQfr-+_yWLd+S1bjA~9`dcmZodB*&z1!5&&mfP~$hFPgvM zGXstenh{64O3z4GX4Vs)(L~ekE+WX4F(*vxR=!)|JK^37%LI{N+3JC zgP=^=ityNHSd;vwDVENangswJdwNZ98l16M)GjI(UL5Fo4PPby9?7sfIODKeg^j3z z$bH{H^WqjN!wSbq)zM=4CLi@oJz0pupjNO#s0VgjUJ!OW`ON*5kAz$U#&_(%>@kmI z!l5nF_3nxyXEo)npP+;CYwW`EUQoOgetw9dQ^#J*j^^6cj|(v91-xum6olX+M>k?N z`Y_MqzkGAidu)cc4vF_Z-2c*uAOA_T#SOf69}ayr$j5AJbj1AXbsqlderqnrMmGuaLsc*Png-@-xoeN)R^*N_u4b7_WLGG4>T9ONGQk#k+ z2_};+jRNxuRJh9dY5z@5A1_L~VF0-lc6-yS`JqfcKYsNU8U@ewXlLhN=&uh3{}y)) z;y{Jjvw#PBY!y$&jMvo1-b{bcry_>@--3JnO2l&l@vN-xlv(?Ox4A71O=1>;^x{P{ z;vsKq0Po$TQQrPRtKw!-`H zEx3KVW+4EF$SWu}53mrPs+CgyZlx-0u8_Ph|=pGT9hk1XwKP^G&F(p1dI|T7fYk ze)tK=MCx0`)UTf&k`w%pbbSrb84!2OpWKr|xM%-tT%{OiQnxQ1)SEd>PdKbA2G_z3 zR>eN89+>_(2s+sEarPEYfHZVx3-|#>k&4I$pUy2adGCB(`iNIu8`&Eg<8Vh^$^2AD zeNNbVcJX_4H6}+{=75z%jZocckC5|^{x5GZe_m0%WB9v?bMl|(MpFRNM9Ju&?x1IH zk8f2`#FJZ6i$Yf1-2;=#WfWLR76zk~@~dkMQ2-;qZGWpOGTLbh?oPP;bZL+UhXGH6 ziXAI$UFfmME8X+XXdOC4sI|X3V4-<+)tAIX`9@1OCSRH^QYwg$LY3EgiLU=aA0eOt zmOuk^Y*|Fp#X)4YI#XUg6IP2}CN+4IxI zrKX<*$NL5}O~(*@udki=9Z}X+AKm0c>g_`WVKayGqobMmrCF{4XwPpDG0zc!jaGPy z`e4LA)ZkxnK4QL%N%(0c-^;~Z-v8k2u7)iA=hZ~S0H%$l;;=*ZkcQY$tMO0zgXm+k z1@&zs^q}}JmRYsvVE_0~Gl`2xk9*(djiaSS?T^U^QEyq@enBWXUL~e{?Eo+zq;=APq2Eb@KS2uU{d5ZPYx0r z@kq=wAdupZR*(Ah5XHgs*cA9pHJEExmscqAjYp;3zw}UtLLr5?s8oAf{Za8NEP0eO z(NlaV?3p)~KWusurTht$PIP5h?J2w42~Mfnm%-xmEQ*t@<^dTz781lhJTk+7X75s3 zt+}sM(Z#ecIFB4qB0GQAl|Q}8l$ABGJ$-a1YKVsJDlmdPH>e$WevE%p)GWSRKtdjE zw;oK1^2IBISGbvwCHr{xB)tx!8ECwsurAQbJpys};I}h2Ba8g*-&oswYFr6(P0XBf9ogQcIyxbFZc8rUU`d`wg;yd(JL)-jhLl zBJb~?Y6eo4;w2jyvoHCAFQ}Cu?K7M8;OB&o%v^9oGZmeJc2SZjYT=F4%!ULJj z|6EU|_B_v=f_%sQLDJ-2grw-)91Pm>Q%e~ekV7A&@w4g|_|@8ZxnBBb5qxP%bjl*L>7bFqk?H(O1=ba~(0rPd3=a%YBWD#H$~o-MkA_5EY!Cw+ z#%4k+`2d&zF(0(*sTY}t1?rd2i!F{3_R-a=uB(VWu{+MQ$%7l;S>iJtuhH9mSS0{c z7t`~==Lc}#Ew@k5Zj5*E#g+bimt3NqLgKYZT|_O!|E2!Rnt~b{<$UlCh8odThUDGy zM}{6(Qzj^ZUK-&q{E+jlCkRb3JIo4o2paC_Y6eQ@MB*pZ#_;>i00Do zAetD4TpVJTh9Q>jXmv^{Z3>g?IlKBPu!+BGAp4D$Wo@TLYg~#)qc(nGagjV@Gymv1 z6qST3EP;16YCIl+U1qMo_Ji>8sXNXaN< zXK5?8)DtNCiuV`%)x5~)6RQ-Zzz^Aat=BX>?6|I19}+a?K$Xj*0y3s22>fIebI*5B z=M9}39+kQhZ)_>uilt2oSV!SaL4H6xW+4V1~EQ?1Y4aNw!OW34RQ@I{C$-6Kx zd-(`kaRZT_Jv~nDDN6L+#SV$T8j5&*+bqpt=jxJZKleX>_v#+Q&Oc87Q8eLQKYU=& zNd72@+spqHJc?~uV_zymJ<=C!4$8L6hNyqPG%m^L}g?3Oz@MupH>6?fE)pp|vPFDA|OJ*)#E%qvI z&XtbLl-a?d1m-VFooq17i@N9$l|}b&Vb;2?)Ib6fUJ|OB#y4e@m_Olo3G089hIt8S zVm{XZdZoWy=m;T+VEd2QALm$4eOKXSLw^~He^5YU&6<|yE}T7D>pV9XiEYDuI5|A2 zFO&aS^4X55e=PGG8dlK@RsPK|lE&Dgbrs~V`Xxd_;=no2RFhSQ>SfN{;}Dl@W{6rMea#{gCA$C#z`RvQ>q^pCkw8=28k+dUiWa!_aLJ7 z(c@O*wyoTUYX)p5Tf9C+d~A!BM+))z4AEO?r)l3^Fr(!9&`DzwGe=Ke#d6ujW#1RL zu>$&W+HXZ}glvrZ4S12xu@C0}wNR`1=+ejjwYfgbI1+URQcTR_m4gE7Ky1fVZ|&+L z|0w;1Mnm5~6?GuZf*zF{+I6rD(;w>?UhO(GSS{-<5JjB4!&|5C`ztGvwWCtk4(@r| zvg?fkA!C8d*;su|u@zOFC734KDk3kyU*XuF?P>2aB2NDAqZOqdBIt|cz$NPEEYL|Y zD=OGkqRbZneyeUVnyd*!2G?3Wx8CJH5pg+f6b~bhuwN=i(;uXFT?zN;Etpd0fsX1f zDr2}le*A)w(WZSOC7H$$pPK;=z)E6|>k4muSx3LQ$ByrA?fY#R456&Fq+OGY!AT1ABQ2>u0AOq?<)hNp|VV%*%nxpt3pyDs95+# zx&tqnfn==Dy6VMr$GU3RFq`J`-4SfETX+hy3Y+dn-!v5WpR-uBjSS=PzJ@C&qr@DP z(N6zK>9UwEF`g1Wiy40i0=S$Er`rvF=HTIhI(K<@5VI~Xzwb+BCuXApBiaGrm$*#r zZ=xqo5_h{i-oG=9JE&c`w@^eMX+@ZH)0A6(MKu>u=NC6j1(>ptaJHje~ zIWK;1fL}H)!sP5_6LkmvbN}zy#6*8&XZqlsmJjhohLjJ7Mdnu@6>!K?P|W372SWN3 zF%e>MmO&9TIPiJeN-^F2^>84CT)c_^q(2pxi>SCoL^Tz3G0glTmcn5dMxM1L?2+2)PA#*+Y_X#K4JGI)5s~m6-qLaj134GW)}-X&qeJYXvHAFg{MQqjc~YL zZA>C!^?zVAMxB92OZiHCjsxa}dDG|MVt&kVX)xzk`Sh=?|6d=a2G!#()RKAq{YDXo zEYcP}p}T{qgqaY#7N1U*;OS-A9xIv99ZMA3gQl@b|idLNxM2wXudBin8E}S2PuZ^r&@?-!&8N2ChWCmo$f+ z*FNo_Fb>lT8s-c;k579TNpsb9^cZ~Q5Y6Gw(X?=7!sS2|MMNsY6S}x%zZuImjsv!#-A2xm%zionxG1^9`(1i!Fl* zLWZfWWWHG4Gy(l}n-P;cwZvw-!jx6MYbl=Xc?FrEl3@VD z7hAImnoi>h>;*JZOk#^svF2i^qd#PS+R}hE*@8Bw1Xd}i|LNQN{N7v|)K#vbIhpWj z;75O@Uz-1apNPc=YdJ*Yho)!+H`fdmkC4(A$eQ||%QoY68j}>6!rT!Q;x6rKu(E)> zOf-OL2)+q(S8Pa&j*eqQ49xBK8r2E2p1S4}|5-fy2?5O>n+2f0{l=J+gr)AI?NL;d za0Mx6gI>&cN#d&csxYz9`~ZAuy?H~HNb(jiQ`9=@0NZ1Nu((`A>wV7fJgNx>BF(ru zxv!WSE!~uHxXAuz{(-5Whh?1a9+5*nHm<*Ii6>gf1BOA2dkmF%;QPg zfh!Y8?7&t=`d)lFTc$1cl=>O*E0_`n=8DQ{p=)~nVmG3mPrMdJtdj^y?6R)FWr6bi z&a%-9hi~EGhCg=F>owhK?UxN4aVWbM1JRSleX6QZz1=sNygJ8vMTvMalBxCmV=PK#pYm1*T+g zauuZ;yP9=~C@XDKLT(*VAOa1CZ*nEQni?km(?;s&eO(GI;e<>5d z-Z*k+mZ7l8Sg8?AYe=;U7YFPqa=m+8dc4`%$qp%7G1!@-Xg`~Z9xbctbLgPFH-|jR zEOsdma23MbOQqexQW*-6x>Y{ud)M^KdRITi@-@;v|FsEAt37H4|G8AQ707?j)<_^Tq@W3UOAkkP=bjzc4b z=9WdEPkzIz!MtcUk-0Bk@-$1(sL@;_L(C3=iJ10xjy(M3JG=izO*E?^F#F9Vs2NQC zw?YJcvN-f`v|O!sO=EUMtKcFG!gl9P^n`_`QyKUBbJaM7#e;?(WrtCWfwT1SYJGWF z49O%Ih2&5}j?SOJm+215%m%W`+z^`5%5jaNJP^2*b2 ztQhOD#3XZ=E7fTsv`8DP>7{n9HURgd#KUg9qy>QMm*m$cfjAP8#x1T)6{x`zsQDHs z@C&NU`FJ030}O2_(!8tTM)+FP`b||Vm?QZLp1|WxhHZv46Wq&;BqRdI!u#W6P^p>` zmue@*-*Yf;Cf>94^7Dg4O?ny{4>=E7WC&^+)!-~lq(l>I7b))18;3NYq?zElS^kWH z7Sf`kP1b|!SRU1*>MRW7t4Sk_Y}8U{A{CSq@vvTXriT;ZwyPE#Jy#t=Ru5H#X++Y?ohnolI3;UJ2XIt6`5HP;VDYbM|}Usv~BKF zPVV3Q-@*NQ!Y}6GqBo0Ge=av0Yj#lYBLW>KOteY)JB{vR@zPFsC^zkegZYs6`((kM zA8iqovK^Kf7D2#_GIzV zjjuE?Os)uNhc%z2a9XwX8A&E^DIG+9f~C~t1n3i=#=k|LQokGSIqCNqf>cq-d+QJc?{ zka2Vz7@lsKHJa=V`z@3I==;HV!9}Y94B-M~>wpgvr1`d}hPaTjCG=iA+lOIoyME zgWX1`w64U5ufCe3QlQsV-bwIyrEKoBJNjKp@VNSG!8AUEtdH2iQaT1IFhkErTeY@% zwkdMjC!q`^-eJVQiq>+J11j4RGwW@_;j*|6m~Eg|t{YETVkQBK$v}q8xAOJ+u8u!E zQ&cc{FKhkVF*|b}K2b(616*G)j^cv=-*5|TPI_E(s7#*3b_|FkM5|jl)H1|nsb78q zH)DC}gf7M#br!Cw*Bt?+(3wQ&wxmz#sL4IG{#|va2QTbY>x`{3JhhV0?<=`K0i_Vr zHw=EBfp(*6{GG)`Dbu*cxYcqc(5q!Hi$44T4%|5h%Z=k(BYO`21+c>A>=AfD&;MeO z$>fv+r?=I~b^s}mY3+I(d z3eA&X!DQQ*NZI+>-yS(-=3Od!8O}=D_#(YmTr%z3EbFKb@`j{ z6SH>jM>3M*=%2(Bz%N502&7zC|9G@@6TU1x4&UCDN9V(TEOREih^H1G6fSJ=XRIwYaKe5IQOG)4B}1D^;RQVb*rbw3Ek| zub^7qlaALZ7b||#YSHC4gWH+5r#En16^39t77~|wE}uu{CLlF%`PYbDuBhNxyM=N_ z^X5s-sdyJw1l2ORi{&R$@w_%=2e+^gM165*rE{rivO)D+QkyjoZIOHH^ibaMMtVC3 ze(zE(NT_|gxj=ui8CAib7u)V`EZ#mnjXFC>939avEDHI)BjuDgf)0ULvRs-HPt2Ra__X{uOY%_urYb z_Sq=FI`$Xu_`a%#{0!dJs|Y8w*LPCdZ$IeGZ?W=wN{6R+BcX|VKgzG3Hut;DEh|cs z5;F4Jgxc=#87x#N&?V?83S1BUvlxjz$reKSx8eBT!`fWFYpjs=2$~BO!jZ-fTx0i@ zF!4_5k0Q7k*5Ig)E0=wJXTw7m7=1j0unQl%;iH(F02QpNy2&@-7cKp%=8wRh5+Vf& zinu!(xXfE`CQ9D_|ZSoF-CIh2y? zm&g})N@f)$dW#Up-c z!RWrh0WMLAoxvjUtM#-VjvEP4gq!gLmsvJQA8Aw)_vC+70@F?J_HsmyBp0KUzwPt* zE}ih))`eNj!?rb{5}Q!6I&j1dJKcD;7HLEz0c~jzCabGu6a>Q}fD-0lO=^8zCN6RA z9O#v5-;L_Nf>??msTXPcS}a#uNB}dB+6s%+twUaEMvDPX`nTQh>a|q#_df>GC}8rj ztL}W%!LfInQ`u1&EL)=9zY`@2krv7_5XcdJFKU>Qmnj7`kEQu`Beq=|3X2x6hV}J)NJH`-P75p5Z*lHGlDY zJem1ZFB6+RHy`vU&MM|s90Yx4R{4INm-nh^6*0>Q!oW%03jG$iOi&saF3s7vDq2g* z4$~Vw$#MR3M(M-;f=zc|Dr!=)`z;J7cBgq_6v0?)EA+$%I1H=3YZp;6icn2l!dqkO z+iQEIwLh=`TMXjxfD$!XNqzYn#>7shigs2hB?vhy)x&E+~p=P%tX> z$P|OsOC3ifx7lLxGtsyvdm=cRHeIdvw#JS73T44A!!so=FrH~(py124T;}y^M)8e! z^VL`1%#`v11805MAQ!;;X#o84yIDKO8n23^NlrY0@^Xy*lBxWi+_($oB${u=%Kith zy+n!o{~CMXRu?ypV|@2<@Q8EM9jAB(J5d`$*MPn(fTbF7z)~Z3YCYZWlB!`GNC8dDC-nzskEC=X8BQNH0YP*=8A}~_ z!L^&W+|=s;ud_AgDo*1_O2&6g79SI<*K4HmzB~)x)#E5d4E_yS1)%@y!}1JX(jWA$ zn1y2J|0$2u=fNLbW_&U$#L$O;y10Vh$!5O76n)PMC1ZchpwAj}IBF@Yag>Yz!b{&x zTkgz?^~y_z`#)3FoU(~_<=PKK-y&KZX{`-{)XDpnpdzLGxt-_IAM+L}$8?o{H6NoFQQJ(G zTeh%boL$_9~YY>ZTkctQdUM z^^)_)s`56wF_gLS9DRIIsNp17)MC2-GJ~ZshbJ+E^f3aYP#PGEvDTe%x&bKHge|-q zk-dSIhOj`0bgmEs1!IsHE}2yRUTf4sD|WfBC}O*#Qt)?x&$%`?^zUlvgcNmN8PbwN~=6RY*gZbma~fxzsIV^L(q;YCwV zd9|LT0*$YPR1BKo$=$9orKG;Z;XJB}tJG1|DV)x+bbm*s*E!L3*Hl%mUYE00j$w;= z$M?QA!Z=^tw>sB6hT-Rl$&bGU$gE#gmf8=sKocIT__g?mC>Z$-&20V~#R)se*kj*z z5cDL6W6Y~`%3VVVcv=z2oJ2WfsM1c{QUW^srDytJIN1DRTvXEK4%pb#(211oDJN4_ zd86LW3!<9k1Q=cIlgannoJe3g**#hX>9#2@pZhU*oQf5QQG~R~F1Aboz;Nzr-Eznm zzJT!dEm0B3@xUv+dd6Z#`zQU3yLM!Y!QZ2AF---FYIE7XVPI+8CaNbzPBiBvCBdAm zx41gs+BVzcriIWCO_F$DYL35?Q6I*-0Qi~IB{upDMXr}umO|jY%`)7M+Fc+O)o97Y zV{+g26w`BfV-mxrf+NWOd6L9X9Kly2_Ow^tLvQOg+){|N{%c^mg8w1=pbva7{Imf= zl0S$~vR>4lrjv4s7sGtvAG+6-Qay%?qp$JYg z3$1`GAlLqcnF}@jUvS6K*1qH3CaKiTGsXQ!PNZx+@v+3!;8BOBIWJngyr{{TA!9OQMIr0t2~6> z32Hr(43@v!`nAYp-57$cl9c0TY@0)KeS+!XNVsGCuk!UHUb|K+Kd^|!Ba#t3i)eMb zq!Ti)HEVgT?#}ZZm!@w*l5AH<5pGoiuS&$Mn5^Htp8kh z?-&O~yi};|l01@uuw##EVzJ(4szR`ANRPRTgld#F_VP6es0PU_=UBaxS@H%yX?DCC z+hl~r28`wZjf4pFmHZn>38@W0-$K|Q3Aq(4pLasf4R0)SF4k(S+nA6W4=FAIzTHf31$sZnVQdG+hHwgy9p(%KTt9Xo>s@#`C_F7#5(?T`u3D>r+YKS zp_O1yvTmgCffll9+x%p90Cc~dkMV!qk-)JZjK z|NfVHLta_G8@qcN_(=)!I-nr$Qv(Zvt3zNR)A_1tX9wM3M>amOVwLJXi<+=gpocDo zT3iZ@T%q8QX7jIR9#m2$j2pD)T=MGqtX-%kqHm{+%fi-s)1scG6d_J{_SL7JiPa~F z%`9t6v3fgM1H@$0f;J&Z_qxhXOYD2?_UO<7#2t%Dnh(fBok*fB=1V60YJY=?LNCgg zT%MXZ)C2PuS-RYaYWLze3VS6=++|XSrI97K2rrrD_k>33!x=LFBq^SX!L^;pY?#O* zFEjBl&A85De>&=Hdl^=Am2zRd2!Pd$~8q#cCQ4s@2mYptJ?0fbH zlk+>CInQsI<6&Y!QTI*`X^>TRxl6;&o`}?;s1qq@zNF^DalxW@-*2k^%1#Bj<()t* zXGZduC&ARw)iibD0R*>%o=dYMmP|TM3WpWzs?%qjACj($!*Z3880^F5_z`>B&%+%$ zXO^UEtP=23-0p~R>40M{ax9NKOL-;FQ5lLoxOQr_XXX_n@J~|Ik9rxgG-K+$F;t08 z@?HN%>3YxNWUsQ4VmSUoce!7C2Yz)|9i>ddXZuE?|9RkFj9 zMe(<=A@;uCB;ukrLW+BB@vf2TeoO%Q4RN%g$P-N?RUQ1+VQO-+!5RM1fhlV#se(6# zrnJ0S<vM8+aRD-GB@#*7LO50fjQ zK${jL{T!JhZNaQ>KVM7SvGhh zGkyU%gzdO4YVLo=FH;hMP60qq8Z!?&ivqn5@vG{SFa9RInqDNQC(T|;#3^PwvO0F0 z77knHpZ>$=URSE@YW>y&e$6Y3NSWx-(rL|b;$-24ASaNo$^?TS2{KJ|M1oK0w!)+5 zAyispvu0_5J2Ea|M#$~H=TTIaV^7}$)W-j(a+0uqQmMuj|Mk=x(5^LywO(t}t#Ao~ zG5jqq!y=UbGfUYuvj9!-c(ybXBvusgJ~=|ubXW`fz~FCz&ZJCiQ70$PI(4qiZIhM! zlaD_P$MZGQg>dum44&du6VfAV(SPbM@fj-2=Rtjl_JMxJpg&?iUFW za}?g0Kv7lse}XH1mZy+P(Ao5UE(5NU51+$&D|5rv(1m>S4s5LeQ_}ETKlV%c-gAh_ z_Zw(tF_iV@@!nPsTC58BUCi*K11Y07#?05PqyR|{&}9Dtb1L29E%JV==+`N(xh+*@ z1~T>N+AnY{0xsw~UCi;7wI9h+cSad?C+oS~dr=oIcO~eX?D+ZC|r^GFES?aU;M|_d9?1{ENI&YOXThXawJGzpbjx_>N(lIi466ePmNrZ>TO-Whn3cEDegmM zDZE%VEL|}D2(Jp5L$!tbi%9PQXIQXOI_^Q|)s@SgCy9MhX!@XAdh`|fahKun=#j0% z{Yj}rUkG|PtdT@?8a}^Lw65_l*EzI)8W#g}9EtR={oeYgaAG&@n7Y%OAAybCxI8Zi z_GagodM(tNk z9{{z3-*$JKc1ds&bZv5 z5Hb#Fx@K0y6ue_2^x7y_0q!MBt1vtXd9>L!jOvF71oE7`nkmfJA0U73hI*VM_pQ<- z_@!uupd44#!_dKLFN7&BW9)4>i_nqDC(C(J^3BT&4Yicm2njRq0Sqhnox?@Ng}M4` znzVsjPah2L64;HBNzHc*w3!Q`xKz8dAS&XnY0G+Vm4aGkoV+ zl|vZgA@0qGOX~I-ef>CKagks6keTZZ(=kz!9|_%AQ64Izh)@U?1$AbG3oipN(GaeM z8nLvfHt?w45DsA+QD~Dzwn6YWT-V1H_TVMjo`mX{n(+vJDor`a1fI|ye~&6_tnX! zb@ievuT!s&x&waxq~_-)M16sH%a6MvTzI725$GY(Sx9#nhGvsdqPHD%^P-+Jpktk9gJ+!3L(-Vai9`PRJ!aY z+vyXr1bC;^{K9Q~U14k};3S~0hK=6JZrLERC{vK(xWp4E%phU(Xj%Rz7IV=1#C~B! z=s64%_U1%@%?GIJYcZ{RZVF#*FkE?_fC|oF)NQs?mZN%c72-+7E-QiDEG>RF*|$I@ zQwC#cjfGQNHvbB74|*0jFyKVCWAdhTwZPWzsT}i29Y5)V{Hl^NH zkm$cJg=NCaVZrcA)El@t_}>k1O@(?TpietpH_hXM5BG|^+DV@a-v81))@Z(kKiio^ zq}V!S#{>>BOcLA6=aE@WTvdxaT^=HY=&~RwTBnm}#ZiB{LZoVA&R<-pU|sC!9*%;U z!|`#p5gi6+mA6J>NJbNGz}w&DWeMw3dhOQ`kp$k}%3vnIuLYGawAQv+OK>rE;H;4Q zE9C5{mI)=k_wI@utm7yRHA47CHL4jPDDeBnevMVfU;3=LkWHffEWUWvFU$M>G-3O2 z+|lBt1pV)r2R@aTnMZMa-tFZ=xbn44fsp7!G>ME<#M{7Zo~>8*58w+4$`O(CE)&X$mH@IQ z)eN7nMj0^yhviuF#6|OK+(k(W0UG9fPh8o4R6gfYu*Hfiy>xTMVI&w)lUFA>c9Us5 z|J&(T1lD0`_Mz<p*!!M%QW9IcF*n z{=^QisM?JxEYWVNPih3-Ee$w*z6jz03?%#4Pv((NWbq+Gq;0k%Dc;#8sjOdFY2dp& zO;uR&YVyLv^iXsP<&F!xXTbl@X5z3w9zCWB=hwB(_k4uxgKvpi~XtZ|Z^b4tYHnPbyIhkF)(qpv*^dALIamjE`9 z1Hmr@d$vZI)q;zyBX7t$BmF!jzVx|Wlj?L0&XAaCclKY9_Vrm`m-9jejY1Y3NS@;X z1JZ<^YIV)^NcEY~3{gTx>NTTnYyW=x+S`$>_OzG%<0xW-|EuVGit>4MZ-b#U= z#5VCb?QtJ}e5i*i;O)%(Ik#Jdv}~`(=7zQ1onU4+p6chwRvjMH20!mcOsf)0XqDuj0hmPj6lv9K8ZRSgEB3(ztn@V_cdF$2|^exZG&jOh=c z+`-8zZo>(U7fgs)1DcY2xj>-m!m^cc*3NjnQhj?I8};3L^QnY%Tjj;B-09?K4ho)s zvqBOx6S1>}8&>6Wv%k#F6>Tlu(#{!8_%Qr)rF$$yKby>!JZwUC*c{UM&!ZIjUyqUpqLCR(`3NSp;E0H{d8r5W(cn~S zTFw2Lx<#KY6a>r7@>FlD+#4oY4=1gDJ%6L-%F;6!y8c%AElmj=?Gi)HFcWvSLy~`+ zvv>UxNz$6YP&f(kw6ao2C{Eduhl6De_F0GUQk_G^@|TVK+|1_!@?0SO;#0`yx0bnGBUil-2(F%X^WN+suJ^xjk&$ zukyif=Xp~wk-tUb&I%b0T!uJqlW^&~m*5Tlk}%SgBiQ)S?NB{yFp?P^;GxldwI1~* zbHJ%p<^5G`Qk6(CQa^TsI<2~B0*)P5 z?VK;n3HhFeLqSZtuH>%K@SaL93x zbcSFm)Fl~LewZd|C|KBR$@E8c3jdqe!yJ6#^`oofFXlrZ0u<02f7ZuF(!cK)P>kRe zhgmW=e*F2*p@$D^b7$xpb%<+o9SAs7zOxi^D71(HAqRW;?Jqq6fy~^XM6`zx6E@Sg z`^1OaNFQA;bONsxc*v2H!!yD3R-IcTTcs|KIv zT@U4Sp(7~~6)MQe4_at1gdrZNIy6$udoTK~ffa-q47h7nOy3Jwz(^gK8kaWjrD8Q! zLRS+GSlBrt(tG#vG1PLR=$AHz`APF~WB65E&g4l9Q^pF47`|@GWRp9V?{=^Rd&~7 zSl04mzYKXvv7Osa=NAg-8^aWU@Q~N9>d>NHqe={c_m-46#C=ONeUmwGq&&z2KjGit zHkYG}pmWW+Qih(wi6r|m#rL!CBu9tOa#AOd9fxn(r+fTa6*V+E3EFHpGpIHk&vLBu zGynS0sGWAZ-!r2s?VWxIUjE6;KYvN`4=hSBwbNuHFc7@R%+pWgQSr^N=#KAn ze7Y;E_iL}FPiP2{F~bW?%Lgg;0lR7 z7V$bvG*~dqGF%&00>)1q$>fj8knUKNU!W8Cj`IMvWKdDkfWfl_7A0q55TZas{BaMn z=RT8~uILxvt@&lAieI@>#W06-q~%wg1`rems1u>$f=@I%t=?Q^J6CM8rH?I;fBoKC zI~nJ83_f{;RnA-MTW@5KjQT-RuDH?SWX#`RWW7%sW>p(^*_=CO!=j`?x5rRn!h==2 z_AIw4*PLd(M)g%YrEC-Q7NBFugEt7_%}|e@XoU%CHSOG(VQCQxHUWW+Olsx zY<1hJ@QwC`KekaS01BE4?c_T?f~7`^RBQwKl{@ou?AsqO2XrmVuf5k0NMhfe^fLobjTE1Bpz&?`x?}TSg@hHO%!|B2ri}i|N1bNQz4oUL(L5p_q-#q=< zqVnyJ-S028|LHJHC?@I~>TNA7N@!g;VdP*~=HWamaxF1`AM0al(BzFXJx-$o(-C_j zol;Nqi*E+0HE}>f!rhRyURk(Qyph)f>((2D`B@pwQ$_9;5xl+Q4@+{S@<%LxLdRH? zobT-Ni)4(K5FE=blz$PR?X?K1-@ex>=d6K}WSP}$*rl8Jidc^r$D1I#k@%8x2n==S zBM9k@bJlQcTjz=r>vQfT8+_@BR@4<+bF-{VkZWZk*27y{ezwKzC*W71{EUfb=i-bK z+&)+mZoT1{=q*m8uaZZq#&w$FU(FjsEtRv^4o z*TjuSSwI5S9Si2QcP!^0dHhc2JpJ?S@6WdXVM3GUAc2vHo2Xp}kK~ndgz((2U9cz- zQ;wj7g-j{UY~x@~sEFVk8R--`o2SdmJDeZ{!Cg`1%^y5^h4lP@lan!;;!#l|>jg4S z_@^gDj??|4d6EMptmsdl@a=(z^&3mgiT-R{4@`@a5?HrP4>$1mz)4UMPZ~%2@uxGH zK!)RJ%AZ-OLa8X@;fcx-y0Md3F((O;(H*^pC5 z+Q=E>tztlTcs6N;hbuiIA-=PfVa%C79aa8=s6*mt#92H`zVV3dU$@JOx`g(HvQR13 zsj-tyz4Hd0Wx`$-njN#KW^^?4S3-vnN_1qe*2+KcP2763d%qx`aZ<1-L0eZ|`I}C7 z2>eo#ps?PjS#F0SRs(ms`~+J7;=GQVsWxxNxi1KxFlvZ(?^XsHa#hsR3cpeY4srzj zKO_^;-wyth$1l?7>8~E~){lSr_V?DbzbHx-!6Kns*HR$Gn1zXT+;W-be_U7~|CcgD zg9Bs3Lwq)2w(tw?#4B-N*z<9YTW?+o*8>j>TW=CMhhS4rk{XUbrST`boJlT`^WaXr z5=W|3hv!V_54uryp)6B?L*)HwvOO!e+s@}!Sk?TsR=c?p)*~evv#CI_1B}nH3~BL> z2ac=GkVh@pu=+S&VeD_*UvFh&2H4;$PqJQ+d6;WP;`pPq>H1d&Lb>z`?iT*^NQWuH z5&a~o;YinilW|HO5FuO$wg$n`+nGZOcznJr^7%8xf zOVDkVUx@|8Ehg9?we<#Ojl8P3D;PNsbM-7TPg&SVnGVCB(25z?(*k-7NZu`CzyrDt z4w=X^XbnN5Andztz#$9yv{#vp zK4+p0f9F`F2c2>1d_3%xO@V_ut&cmZ{IUIe`B6H+ZrHcqmfrHH)vViYgO}*nfD}aKI6`iT$7Su&%v!50*;1TgNCfiMHG9EXTB0BYU`nT4)BAmF#msK|H z#PL?j6`6FN0qaos35$>Vc8dPHaoAu9lt?9Ax&S01p)w_WA#Rkv7A#w5TXs~UOa)Ms z47Q3gzY&QvBi{D(|F@g}4}A*<@q8M{Nkii!V3@cS6!)RcmX@`kKgAyYsu*}Gzk zZTQW6tJ}T{mZ5vBb2+x1!g{2D2}5xx_Z(0~xvWC|bP#e53tND(5;d>0@Tdf#yjjRw z{9r{}*rm|QCl9kxADeDndUOr-Oj&>sS?^wl`bhnVbg6IBG>cWH(s|reis;n)@amuK z&)Xii-Yl^?Zg=L^8!qOt+$a;bRL3@%&UZ)PPZyQ$ScJqM9=XcTe6xS;tJ!bsq1-A0 z-TU;w;u*<{JadsQKmKOw%_&RX^4872^7PNsKk|(~*Y=kvN?NefG$%~BRmRaGekKU6 zCfikAtG7mSvNCcxyxD2+3Q_PPSen`>I4-xpBOR6X>hKK9)Pa z6&o;>bG9URy9Xk%t+w(=xr6XVTnD`57Fd*En^9rc&Oja7B@{_?Prc)M(Zn+Xcm&x2 z_>ZLhB}IsM;~N=Kmq9t^!lI;-7A2tT87GamGAK3lv_gaC_+uMq8n*L72%_wFk39Xn z&7Ae3jT%10ZocmQ*0~GLr2%bRPRu9~X;HETH%4$m!M*+6%KQcawu=ldWL$YabpKBX zt^A{I-(vog#~&Jcp8n_&`S$nM(*GhK>{;FwEWr6Ic_PQ3S{wp?`!9ZH>o;t%OU{{Y z=bd#bLtub_agqmr;xFNK&X}1r=i~7h)Ct!Erw?=G)|+AA>wrcX2jR&RKjYh(se%){ zG4f&IPr;7q0yM{)Ao9@o8Q;eeKgy5M1$aG=`sF7VcYwccXSMBKw9&Rb{syc^s<6ek z4i=%svi_uu;b&)2!VC{+$u>;6gJ-D~V2fz!klvCnsc0#LJl6@6bk@J*QjRxt02UXH zy5L5fLGM1v`kXUCVka#;*q8<_pb?(Xfq_)aR-*;_GXM~V zr{EwT-0(~kCU9%;k7*d&6ZxRyNjTyNSj6Ah$TLbh!yFHa5+u+>-lGWbu_%9v z#MP!}aXoN7wvUuzB03$45)P!q@7jD8!0Djk)6ME}Vx&PTUiyb*h-u{C%%j<7CQ6Qg zv{xFxXB9#O_S~*~dWo%lZmHE^Uuba|Ce_Fz_hmC}tXCdlMr8)K^)|r5=!9!dx8Y}u zv0V!{*s9x}#DuyLX?$?zHk^7W9fn?enhm<-1Sv{rvP^+x{fnz?`I8W6_F(@nR%kex zCLXV62qJYj%K~*f?39r<`uxe(1^H>h86p&FVqNDlMOHb}g%wo}kNzwdS1TG}qPpak zhi&iDEwFsSEj}~_<>a1)Mww3f{s(mpa!2D2SxsIoiqXuY*{9OcPl_kxSJwkq;d)@u z>O zB^6>j72%PNq}M`Nlx&7Y3EL63-t?#_(-t*Wa(nwfnSy2;%UVK`n4O}os zK-#Nq{1JM9mHBMpk%k$g$uCDh+UrR1Grse4mOUsug7paT?q9dlwm-YvcFkL3bz7mB z$5>T>+lnZEFfEfuA`1fQfbRpCgd5~YDq6aY8)UtvkG0kDvJB)|DQf~t@V;%My>zZC1d}7TVY5A3&KdEx$ z_*67SiSfdl_+~*-(g3-Jhq<14(gev+?7t*T%$ES9ej=E}tC>f$Peqeoj)1h+;gp}- z+qj_Ox8~I#gMiA9O} zh`LIe6?%ut9R}ZYut!QfE2Stw1pIS`<#0Kf;sBBEwCb2j&-l)a2|7!Q!4V7~7^=Z6 zLGJ#-gD~v^1#k)q66%G_aHtx#-r#U5CMNSGV|7eI9DazkW6JN4@GC~-U)*~091hB= z$4YxCge4y4s{A0oj`djKPnQvH(K21m7~VF1-*1g85MN(!bAL3;ia=AIL86o0Lm715 z2m}*er+WZ5nH1oF(=HXbv1Zy>9pqZlt<#cl94;z2san}F%p{!yEIJg-{ zhpBqI!LJtmiQsD6VzZZ+;XZzsG>-(+ZKY9G5d!GLJ_V?G-{|cx+C?Rhg(6@&Ub3FhC zMB|A=@_%89KjhMm+kMjCX?KX9!;74jFp}RxjXwcYOMLW6BS^bLqTt2xgpbBo@Elsx zG-2E?pam-2ltcF67QIdPzi#`NZMJ&cUc*J0LRvSXf+T++s^kq!i^RKGd4P2QR<86f z?pAIUlLp$LOHZ(_Lwou4z^Gd;EU><6wpH7PU%q0Mudc*5w(XQa)|3pu`t*`;3_6I# zARY?a@9asCb*5^ON%<$bPzn}X@`60v*C9l=BIfjpo=zG9LYijDynxxBE}tMel03_; zH@E`17_!}rlVMSU0%W(vj>qp1J=Xy`Ug__&JAYjs1Rqx1^J(C!s+kj z&5!?Y&HPJ!C_nzI3Fq4%$?)gh|D-6<#D%q~Co-P!E&k+8s#!>2V;+`C>*;pkY>CiM z5EN6w>8-|@w|Wyo5gZKU;Gc?1;tD)|IQR)58J|2AkQLWYz{U(Hs{jtwq6Y|r`j z_ZQp$I2KXt=uyTi&N;zEz2k?~{`#BC?G7qRU`>Vdg)>ud0 zN_l`|mKdmpElM~3z*i4>CxhIggx3Q%84q(ke%x>*Ll}|rQ9}I0Z}&}p-_G$z*%N(Y zshX9NpX^8~oo6{6Q!<~5248BWs2HFif~R5EUfZ>3i*0*if$dwq4O@5pVEF=U<0Vm| zB?4s(Qk%kj6Yfl9y>Uxz6V5y<$APS6!+YD%55H5}YY3Onl@^*Po}ESc4f|;XVLPl} zH{CnWYS!ZhHz>i2N)CwWuk3`~J^xXnTWX3!M@tg{Afy}{yK=r1Ijr->;E7P;pq3!#MmViMNBw>5m| ziE#;tZNuM(GyfohJbu!YkN-MJBTeZIe}3YXr$4EbZ-0NP{ZEP#wc`{smXoa-^>(;V z>@Ae#DGI8u|xdY(UE%Ae57Y4pBujm`GyR(tmjMYm4XBhhHb2A zw7wTlwo%uf;dDVpk~ShC0vY9ih`rvcpIwRni*>F^XV8*uO#&vuNz(=#4BMqgsSQ7U ztc|^Z`)sLD;e;K8W`n`wzi=F8`IRXA5C_j-DTIPx$u}Rd-8i7Oh=;(jjQA5SjMyi9 z`vcY)3Z=9#N_2eMgWn-!ezRa^ePrT9S2~#GU_Rz8_FHfG5w8bUyA8C8Y4uhjwiVR7 zEOrX5uD;%W^S~4K;=I>T`a-PgfCd!$*go)&IB&ZC{Wf;wFsB8+S+^4C5#jaaIHP2H zm4w`hBL`Xc@^a(qtZp*mqyhbyFqj?ugPRVvsql?zm7I(pzJ)O|4NpMNtS5nk@^r#s zz!C;>D75X)!gD`cIVxHu+CUS@au#j*Pv~4 zn*%3bXT9?jd(S23psg1`NzAbldHb4Bj@^Fwcjq1h91KQaa)ibwaq%<5nIMReQQ<${ z3l^`D89$X1+V0}qQxHO~1j~PviSQls5c3wY*dlgFlTMn7!_kKmDGK_}&c*&uoXt_f ztvArt!%O){n0BCOCNFfg6F+by^cUcmQKz0$(lMq>=;oG=o8vWwh(fH&YWK36>^UNx>hXvw(CTgdoCeYbXFzFW+j*zB|*JYH;8$R@cNd z;Llrc`kXz^M&I}@coDFG<4p+22mX;>lt20G*}U5pV)bUvx*Y_8Z9ot}8Gw%{fHq<9 z>^7*ko%rF?tpY1Jvh>i6aX$1NZT%~MK*lDw>|2l7?pIgCqJ*c~QyGJ@gH}x_+m!#h z#=3MbmC}F;6n~gY7NTwXr^~N`AxTUb0EO@1+dME8skQoddUmpTuqdg-9&1V#JfozH zXOv*l>+RAJGq*g9S(F2f*`8US+ioGY=V=G-?xXMzJZ_Tu9WalKdu;e1b)z4 z_-VpT1>Yg@7wG&0ZvE;A{A_J?Sn*~(91ecQp`{#eeCR0r^-#cZbu;3pK*|d}S(|*) z^k;JdLdt9fZFDXB+TfMGhcr0&SI_4ZaK7 zUuPbC-9h+yKw>TWAYpUje@OhGe`lN*)nWM2_G{4x4~3u2fwzota@QFv;o?#7V?OvC zP5(zKjd`^Cpz<|W`Nnb0TF_7AK}46ejXOxd?N!0NblxD8y5X+V~IbOCuwL& z>FCHbyqM1zpY$hH*hlKXr>oJ2c#ypM3o6PFaRpR#4z+&)zz_QoXnL&XAFL?hL_PD# z5)+#j7$nX9L%N6r$a1`9SAY; zsfHB%;lPb1f)xkz6w(r)z(&}F zn|fODPQ_^3>4PaPN)Ad<;%Ah&(uR{U6^M+x6-2Fgw~aIvnw6d(e3E+hQ2mC8q9Y=Z zJoI7~F;u!x0D)n{oHh2^Jug}@ZW1CK(y_J*hqZp-G#mELF*>M@emu;TZy(GE$m3S8 z+<~n(kJ>?K|0(@DTGRmsx}Il@v@xGLFSG|G@NjP+$~f~!D-;fheAdCjWfcyW-HXH7 zI@8=0ez7fZ@=^;k+OlqCcFM=kvWouQps&GXi-M0H$F5&@2LH?}m$%?UxZ+9oOQ9&K zn!DNxaV4azV}Mf%6Dmr+`F<AQC=Qk3ri_% z@msNat?k{gt`NWfQ>f>2 zc0sv4g55j@rbe1nye8$WqXA$ZO@oQt#p|9Pl%e2%97IsSvzqi0W>h(6f42;-PQ zwt*jQaKo0(wr>5Pwge>r06+jqL_t($DQ(sKkcP>SHQT&xhrK-Sbxfpb0qK^9*o1+K zrt+DV4F$Hpw#Ekb@27K1#)0lpNzC*u9`ZmSNp8KtXH=9#nS*JXmtSxacpj?XtcV$G zt$1f59Kw~Ao*#TV5WfzE2d~`lhxxeX_a&V@!mYogJgriKMO`T~QSkt)5yr)!!%03BD7iQsTy*G z&iMhsoW`InX&eg7F`oS~vwenPBDKefLI9x$#Qt&^q<@ltan#9U7y<mdCDge!Lagel0~{KvVo;-KGqA8a0{**CY|@DX2OBDg%QyY-S>sBT|r z&n(ALls~2yCj5{w`(du})*C;}m1Xl>g!06M>32KvJABeJ4i{)a8ghC-yaa=#sa&8e^|84J?8B7(3qh;fA**tr_XL2NI~ zVV8e-=m>p#}_JSa*kaYhN_ zPvaRSvM51{6#Xdx#`4qup{Jj-nNPl84Op$~Tv&{W3~PsJm?i!Y8tS}>=gS<#B%l@( zPWYiOsK2JEf#^k;4UtR-RBx^b8#R2Wec(#0utHhkXXJo}0*xR1<1IslsLLHYciXT2 z@SttnxK(?V9XInqblz4xOH9`izvmnAm}NeQL#55oK?8-qZ{I#S@slRBs;L6)etrK# zwrb6K@tTgn&&lrq;JxNOmq=NVtNaqm(Rb<~ApYioM{V)4l~5`)CjG~Ya^&TAMOG*4 z)1#-o|H_MO%!pxvt8F+b`V&9vqXDa~_dWc$z4ivLKPP^Qqy8KX{UiP!6&3d0cb{*Q zCybM_hBF57qtFo5CW!|1_Q1@i?G;Q!xmTV2Fh1?U&qIaZeZe_4^~B@a!0&wRv1gvQ zXJ4EPD-;|6$t@V+Ia>T|ie;r;?ZR`;wCSgv1pGJ;Do6R*?|6XWtQY6l6SH2B1xQEX zFU4WZ=S)A%&Y5vK+CR$F3Vtf}UV3$bJ^aKhtAV1D2b0F9J>};)N7GI@$u2qnEcNwP z@#AK#*WP&3{`km~wigORPM+h_9{j~nmc9dZf91vJf_;ELfiI-nAKM+v%A38GEMJM6 zzF=KgRSoSD|1yDXyE0&p9sYx~_`uk) z95?AzzPQYaU>QOgfDh?ORz#ge6XcYGI0wx&Zk?>_sJ_ODTt-lvgPVPm@<+{G!tX>2X+=N> z6!4IgUTcv{i*jq z@a*PWZ@lq>J2)c<3?ld@FJ}hHXSM@@Zczenjs$?A?vcO@(GjFjF@3u5=oLDXJk+~9 zJp<9(7(GsyJ(KiTS&7^$~K9SH!POBaf;s;MTSZF5{t;+uzt? zn_pXR15X)deaH2;jZj9+|Mg3`!O!oi^QV+PS+>M?+hyKst)!+9mL`d}D>j<)`Zzf-U%S$^x2w)2H0P?VH; z8G+9Qk3yUL_4imw-)@;kpj@)VY#2w6KR^()7C#J(574xHGWc2l6f$({4J37flkJ+;?&67aSttQ&rEnMvDo&ioQAQh_pGY_2jyMPz*OYBg8rVrtJ zbrAx5@VysSM;D$m9g~K$Y!8&L++LIJ|D3#4ly|j1KKhJ3fr)C@u3c3g;uDR5UvK;Z zoYKODKjaf0dV5tcfwzCg#b3mK<~yg_)t6mldu#Sd;$3J`(6d<&!a79%bf7jAdTZr=)es<3vq_kok`GbtM6Mqd9K*NU(w3}|Y zR{fTBNNcb(eoj7DuH9gF{_HobHB4Hg614sJX(iJiRt-1b_(4n@aGL^FsMGk#(~66W zZOgXpcKeU+(N-NEg4hw|NA98YXt3_!E5y$?6;jRdbGyfGDC++EuDhWm*pEKharn7P z%k5M=h`Aq>cJ;8@X^x-g5!GVb1M%;utVDcV7cWlIKKir#94o1q_{2x9wh^%IVISy> zF(lT%woa?Zn)+>P}Yr+wOSfK@o@a1;{DE_{nnPQ zS`GYukgM_#+COv1F;$aeWfwna%$O>pPqEcvd_?PIbRBFCMGqCD7oRu7-hJMgqNZYn zveM1;~P?5zOICfoAo$8u{dNy*~tpW2hw{$&QSHU+#yVx0+usU$Kt0_ zgvx~#w?1V%URZ2J-KZ#mtN~sr3WH~H5#}H)R*QOId(HTP*8ig8tQ?AxPTVHPECU7z zFM6`<8jC2D z{k!94oKvjtG{{kSxG)2u26+r|;g?fkauoDW@{6)lQNn)C>w#xX^;>VeJ&+#$gc^*W z+3%ykpA7(Chui*GSBOqNwQWZgXip-OM@5m{pzlWZH~O!bteQGvOu+t{8Gns6~&>r8&Ky~0=YjPE3NbTmOxQY*&5Ntgxz2EnZ)S?TZ*XRC5%0y#yd@SvWe z7Y2Xy8uB|Pp+hV%FeCWrAr>wSs8ho6+EScRfo2&b~ac0T$w?2XV6{)?jy6bRe z&iIR9QF8I|I)Ie>fBi{OC*=%*&8j4Wp9hV0f^|dzuFzX{+f%ml`K8+9EGrXaiB=IX z2Di!o0Y%Ax?#y5q5CAy~PwFG(JQRL4-W>7MWs1S^9bu;NOOOPWs3@u21&b1Fy`iFn z7A4GRn*N+H@Ene6Z0-2p-~2wMHPE|Ahpti}U`^;CTbPrx?CPlWRAPM%`dz5jCT#ovRI_+hq=FvWv|tL!~{bi*Nuzqk1d-$cH7 zaBi4J5f^z5(eOD6{5&6F;`q__$?LAMs=d`#T+|YOub$oP9$bg~%nNg@TRD_vpc6y=p+g4QXaD9SP#W#SB(Wv_9w`4qk3VA%&Yb0E8(@;fGL!V}!OyNm>y+NT zdfDf1x?Vh#lQ_ao=nuZyy`s!!zc3G5upSO)D>NkSrakzbk2hHvZV>sx-`=Rc?ek8A zuyDu-mCxl+6wQYv%a70oNPk)r#CChM_(eUoBOog3Tws5H(~Zan&h4QoKigD-zX?}q z7i-JSZFl|BDqw+f5co9#ZU_A%e)cGyukg7~TxWf8{=@+&O49gIP@Rdg727(#efv+O z)L{E>Z~Ke*sTiwksI$NQ=!b0Nut8Rf$tHPqEI;w{91SXHZn^a?-9kpLq3w07g`@0$ z5kDup+>Y|m55CV%m^fC7L$1)L@l%&eh2ht~_djT(I8TO6gTH7W9f6LMz*WI(KCh&MEoC^;e@AvAuY| zir}fZ-~qm?*KM@_`T1`V$5;2047Q{G*&eB=8;El{ZvN=C*mg*Z67mkVS$~M1he&VR zzEc(rRIC+a;*&&ZCw^Mg^y<;wZo2UURt_Z)6<5viQ|Y@0efRbs|H`WN>=iva0zbQR zEe_M|iZ*@Ijn`VA-aVzbYk?n%5>Bjs0Q_6G?bJc#-kF_7N8A61pDV>h=ue-x{u&!P zcz|x#YK~t+5Zc`je|itv!6qoKU{NAw(ti9rPmC5LA4U70Fm@FBAP<^0?%0E?N6+}Nn?iTFnEpyqC4wA zol%iP=0si8Po!XogO?MYNsfO0;Z|bIJn-c4lbE1Ip8lE6k#B$Mxql}84`aAEIeT8*!x+}Aa=;RES4 z84AVWi;SHsjtiEqk>;5K&uRXk)yc>&{| zJ@-u=G*<#)`pl`YC<)OcyUsvm03RZQI}YF}@F}OqiE(2FnY`JPq#;PIdximg=ZfvN zX4VQ@`}}g-vuO{|`UHTA08Vt7&lz93z=oYZ$~Ml2Maey{$eMz=%7Zq8?TC~S;7aVWnE;$~n!mgNUZ~Y&HP*VVl zm=(7_W7}uF0YynUOnhOQj@3Z5FDMWuK|m`V%#|Q-18Mqa+IqT@Vhq0d9J~4_e%347 zCKV++Y2lL`<_e)rdFIR*kwW{EfQdM)ubEH|JoLnD2=(={w16Fj$}J%-=AHeS%n-%k zq?6WE#OHOcNX$S&AL_HslxuxI;ZLXuHAdE z9q18Tvv#AOC7~FIgorF-Oq0|D%LZ>f0Y4*Az)=YCqwNnLHpH&I;v!8}X%W^OKS{&( zzi;0@`|SfWar4hAZBY?I0&fR?Duo6Q7=SZMF1CT#_CTw+*6{OoockVr(iXnCRQOfq zcHrmaj$0o3_3dX@UwMIz7(7VVn&-yvEBg08`i#weeUal=B_4hGiJx@sg~K$jy8Hqg zk9NQtV5n%*A<+QM`X~KWk#Ilq)N}UYoL6;A4BLgCqojYt-yLV9Tye>{cEY3yZQ#e` z7R+SkvoG7M=U&!LL7eEc7eC8YURG|GVzPYN$;WFOO0M#gj~Ag|JU{n!d+e#%R)g(Q z9fiLu&Pci7?04FX)8El1{?5QZXTh7YY}yCw5-LtRqWoRDl-j$dy~8e;@lL;eDo6dZ z{5&IN;nEfMhlgfbHIzlkVXp0{f6~9Wa~FH(DO2py^Ugv$BQWp?D2Cut`=g5%Kg7R$ z^?Lgq&I#HLUc`3Eg%O>%gt%-C+_J-MOyqx(T>@_;Ft8SZrxhJn;9zK0py>2$DaV-D>nGGe8pHEE%{2uUlL^2t2jq5ZoN5~ zTW>&iwokOpH26$v;)!4J-4b^?R&)HB*fBldeA;e*gv0#Jdt-&|gbYge^PS1Qih)n&DUaXfy;!y0z(T!~Cnf=KMt5uu{Puqd|Me*7%cy*g+x{RD zAHE_;I`j0;w?DKK@#Cazz@c`7cPs4=JR32RK(?&hPpYk@B1GT#&IIv{5;3aKcrnhS z%`#`HJcSQ3Hx5yed1kb{h`=NO@8dKb!7m0VUGPL)MrWJ@5MGoBpG$EqUlmqH`@%&2 z9g}e@3xzmDq#1d{8B#KJ&L=^e6&iixJ1a0LKS!A+3}~F_J7kfK%`=i0odo-Cq0o9NV^dYq&KAS1aOF zel;A{9Y}N#);PRT;+(HsY$Iljv5nYgyWrPzpyVJrK*hiM>N*>B?j)OZ*@;$z{qm1| z@jk0sk6UQEc9vzwZd~Pi>W9y;sh^r|`ys3>|JI|}r@LE|VdV~+NvS9)!2bHFKfI2B zV;0mXOxV;WC~20G2w-fB6v#b z=`(O!x|+uzu6UU1s{fg7+n$7?gnOeQ91}7RXQN;-@t-e;$!sr*y7CufrUml$;P)KH zT*Y^GzGMFbwygSR+DxJ z8-q7(ly>!PFKp)$J|hygHx%~lX{X`5hu)wO!vp;(&~iTo>mO$GHJBt&vBfwvJ0CY{ zxK@Z8sN7;HSivG3e*l++#hWI!G>c!oexoL;#HX^c4I^Cqhn`G~{A*Ap-y>?2gO)9X zBiM*X{5;HV?8xETPSF4*MK1hIz_`TE^F-DF|Na{1eWF=A@blaQ9%kFCdxg~>3O^>Q zwNTQnUb_KSAb#sjd+<|G=gKw}MSXkqFs`h(RemZ5d59`4JF2n$KuRo9`zXs#{Omu& zuCbG&-ca>s`y+lTw$^RjV%vB4tvBs0Kk@g&`79%b;MO0}!xhXz zodZ9&;B45mRaPagH1+<bfc?3ZJ9=^kje8ymgyx#;rSz z82{VT{~iB9>yAUD$Kbm10+h|I1zX3@g_y<5F*d%rTvjIidRACpKbZfNtGMI`OQGCRwZugHrOt@5NXM0ts@H$^TK{CqEG#2p8>=uvsU_yjx*F zslMiCFF@h2(296PJ>sYi@h*fV>v8{i4P+5mp3Pon3x7A)hQ4#OjXiIYEM7VOG(|;} z@-0%rm3c&_%x7>#-}ugskgNR2w&=hVd?YWRZPZuQ+Q$1{wH>pTsNEN0OD~JSsuvve zZI-qXH{*=^(#6&jx8rEr31wKXM1sqDa^Y75z-50TyjcH|xrqyZvv55y&!gft*BMyM zV;f4-KZAs&Ae8vdgwS;)_(N{80i?cK!2ldiqj_(v(qZ(>s}v<@1Kf@$>oY`ptLGo2 zZ!AA2?|J;JVYN8b8J+q9ZPdp3rAlVsjxo^y7zzH~Ug-#aF+k~pX9CmX=jl({f|mLA z*Y5rY8nQWsMvHf*A!hk9q}3DLNzWF~qRn>c~28Uw}?9&~4=y6pm9)ThvJW3{4`XkFX zhUR?7nCbEd&>o+YIgDpI%oU0fPWnn|-J)$kPLn7EWHJ+>=0ut08DqAJKSl_tv@(wj zHh?Bi{oBK~_Qh3Jiaorv9^r}Xst*VuO*n(24rZTS%8IR|XPHg=`*UpIlp(ee2fHr7 zfn;3C=Y*4GtE;ZH5ob-bIASe|qN+(N9j)!_V#6F)H3rvA-ztApl$ z**6}xnhm>f`g>Ot&B5S za4=)cQ5cY!xA>&fu?`(S8dfX#DL}IQNBq34f)my>98OsLITHM!Golke1-`?y{Y4~V2Y z`C2RVXZgEyDT3t$C;jcGKRCCncqHb5EK_;FUZc)6|3!gcm5K2#EmEPRASH*ZJX`6uCC5bIcbVr z3k$9?j3qwN{F!i&@c@uP91}SCKQYU(-0_{nj1htd>spYJKe|;W^jSVti#FK$-_Ei9*!HXiD@0V= zMlF%AH=-}}nmWS9{N4G`ngB4ZQWPg4QzTcwhF3HEnE-b&IMR##`&n$gsRhsJB1OqW z(NTG3UF3Ko_*m}vPNK#LiAT00D>(T`9NCeZeGwPS|0>Q{+X)$y!-Tfpz=}v$YlBy* z{E8MvdXxi{N0Gfe^v#qEVM+qPf{kz1Q6zG}PkN-~4?$FXPsi^BDE;vWryPo%$A66J zuQtoE3gd!T2jA>-loi?J^Yzc>$gT98f014sL)iW))B0GCR-SMFqyedudHBH&t(FzM zv-}ucl+2{br^5Lbywd1p2h{-xLu7;37s!8v#(3b_aj4GAf@5sGq%Z*S%TEg-7U zth^yE{&97o0)r$4I1Wm>;xbpJBdCBQMoH&Jy(|kyjMdtk`DgZcec3^K4u`o?_$-A9 zuAi4e=pN2@7A|_=$?`i)`g_Vu>cja&W6N)O1c$m-S^?i#!8z5MM*;q;G)Cxht~s-c^52Rt~gl-{t;h1r-b>Bk7EhAf@hXr z3V%Fpf6d)5+LlLOw_@DB!TsuDI>-wZC1byMv2`2WPfUk-^Y*WjM3h7V5ab+@{fuzzVCbZDSPIn zxq#3~eT|b?u67Fud~d=z6XS5rEDVn&WYC}Z`NzpMtv2@T!%axcoHB_zV*Meu&Zc0l zm2spC{h!R74;5l;z{2TX0-^Tei9BA5@-wl6mC}#(i~Mkl1GC^VdF%KIG!7_#iTFuY z(Sq$d5(@7~|6^Ky0QdSwdt*M5xyvE&D^uQt+ExF;&-Q?l=kfoU(4W1Ro&2q>e=;pD z9BkTFX=^ub6CWBqtiO#K)L+(x+)}e(@p8NC?)zj#2^Pp^rM}hBSdX+txZP`-U3~U5 z_AX5FoF}p=`>@7#2!=%dBnHV4_mf4*8&s4ug~MEj3qQx97$=Dx=)#a(k3;m|7@p;q zg+In<>z#om7k<*?PxF-I-z>i?j}QtzFC`T^Klu>LxEl*Bn-{LPtxzh|RqnUxe}9pc z^{ueA8iyC)EVOv*6KKcr3FGqQufJD^t*?K1wJrMHD>~?%@-2A)`&{2`ls`pFjv*S0(T18JAFlbuOSTi%FJ9F=&>7p3(MEWEaK-px z*wT9`ENQS11f^ixPaF7C38Ngq@<03Po7j4T?PO3op9PB&Z+Dato&K5jmjckL^YHM; z|IHL7&LGsN5@+Jb_QqG_S&ESz4f|%p86`|5#-rq7Tuhhv(Jf|1FEA60?fP+l zu}Q|k1BNJegEUDarw{y@mSEHw5tSVzKD>p3jFYRK%nIYsAto9l>CH66jVEy?j%;s6 zK@bNr#mJ7PxbY{N$m%fHdD#Dntv98mIHLsD19O1l!}6q%rkvS;(_e>+KLt#!0s*935e|eLdDdtfdCqw2KY0LiCuvPr8)C&}+Z!981bIo6 zQawNjJOIm+fu|0G)d&PV(w<5Iq}=}I7JKa%FWV{%`o~>$l1=;Ed9q|#{Pl;dcJ=mP zQG!m0_`Jfe0Egn8eCG|SCmqs;y~o0jUp{S>Uu^?)tU|fw+=a6YyZb>5D*8YBuZTZ8i63}A)d=D}#O1=n-lCgb4TOmc-dSYtC zNt}s83DaLkTmR`CL;)!yXy7nc-g*P!(&@?AnS5vPL!B0MAnHkEs!*K0JpT>*-Gei2 z?_M4%2@4w}ppZu)_it~!Mq6oOz6qHM7Os@lV5MyP&fV4kI6YzYrMh9i&Y(s%4!{44 zeni0MZQwt7{7(8j{qybbuc7}@X8-SV+i?RLtZr!Sdderyv5I~bw(hqt+m;7k*H#-TqL5MxW3cj? zj6>B+piJ2~f33a#D{LDMx)e_S>#?ZA1DOY(Jlw{=`vky+H6<&8m_>TzArv-Ul8z{U zl>dYa^%UiX>Xln<**9@BAgtMVE|kV;C{Y@qC@CL3$d3E>D{$R0loy?mH;$9(j49{o zpEd&((d7Uy=IrEp;GifW5sp#$)42o^e1-u(aV8GWXC{WSq%iD->_mArx?QFdiLO94bOB7S=Q;mH~9X$1}Oq6$rL04>-2Lq!SL-){b= z`A4e!Y}8(*C=veUDt}zyLeWJxwGYT?q+VO)mrRIqCp1&L!8iSo2lYz=jWSszDkCs4 z^NE-03b+U-!h%D9n3R6(qKMrl{kbU2!|u5dSA#9l2V`B68-Mfq7c#(49^hp8j|9Kc z##Sj_f?hQla?5FR+ibohyEIRk>5iS#IfyO7Qs zoxj=ewDQ2v2s7Yk$1BC*Xr&Nj)b7>bLbL?7z$ zVI_k~n;BG;5DP0Q6fzAj?QRLyL)(s%NtKQ*Api_%A8x04{Ga}Svq|>BY#4j|d59ct zsk!u9@3qq2Why_p5u1os?7`*8RJXpd$>!hv9Nd1eChM^t+js_?G~7=5;OXFXJdXw2 z1#s6P4_PB~eC-#nSRI7IQ$IdU0^Xbd@sQQ5*lGnh!-Vk3e6_C_`51EbJFNd%YrPaX!^PVro3DJI1k%!ToK2rq@X8@lFwgg-Np}4*}$vuI6~z+o%v8zX|J?91gBme z+IE~w^yabkjUTo@_D@-q?1G>H`f=+G6(uwekAq?cQgmfQBphwU0kd7dX^Y*B>vC7E z*@#@7DVN+s%LS_%lKcYmYU?Ht|;R|2+Nk^p8UIv1)(dernM?1asro z9T3br+1TNOod1#efycf6#!_5kelNC0@o+C}>%#%e6UL6PYu|H;4Zzt=exp&L6dV%E z$RiOm_d|Y%5X9kaA9LPVrtg8e9=N%pgxEOp_%GqfmHDKXp0rom9m6wk4{bY6>#>zC zB<9$`f8I~Y9o)^}THL0$6ABWZEwUfCR(Le)^o1^uShQjL{rUG>Jw7;lfJSKHYHE8n)w{ zI+~f#RlZt80zl@oE%`$@TlryG4n=>~xtaEF4?sv4tu8#e^(=7 z2BAn$e5FjvMQC11mfTwbK^IZB;gkerAdQ(I*ZfCm37&-qE?P;+q7!m0(^1}z{D=7# zO+h2CH>Tlneb^cZn$oJC=Z#T5=LSv6dqPe4c$7nU6AnwEbNJByNMoL-)4=v90);w4 zc_V(xUu=t1%&?9ajkkdjY8}rX#G7$JaJ4h6)v6psro6IVMGK^(auE4#B)tij`9xCA z<@68?{zN+gVBet+;;^m}pKXoj+|9=U^LwiI*px{Vty{MW9g5!x@BzZ{J4Af&+hG8R zyGa)U)uRaRVq3%?6)}7>Z#+6c{S>%K2hNq47qw+1Wz8#pavi<=!!j%LpatiYq%+H_ z`55WPg7E=8SSZpKq-_;H%fNctxuen+E?H^=2lTT^BPZq%BGNtkvJ1a65Ab3=38&PqTG*eR+~@0_!dUK z@aRiXfDIqS=@~*yb^!NejVZETp;IZY05Bne;4N70X5%fx#EbXBs}rduDS>b&_>-@+ z=ZutX@qV1Fh~6j!4UWmT5bsIyY1@|!plN3&F|`7;{ALh#-l z`)uYv{n7TUuhNPN_rh~!=E~b~LOk;4+mw(i(hD8l_tF-@FT7ZOxB7~zkdnu82)GkN1*{%CtwZnq*FuttGLgB5Z|WkLf4ziuat zuz@&uwdV;#z_5`Tv}4^nLzNRh+2xFilD%8^*qk5F)*faGtQ001_cqwTlZM(!A9<(p zq7IgDl!PSB=;-unt^LPq2{&LBYtwIE!73HE3$Yf#Fp)KG@f!Vi=i%TrlmUabA3hiJ z9N&&I`zxJ|*m_bi5@+H_CxE*(e@2U8{&qTa()D_kIE0t|Y#J;|c*UiMF$HTcrb~g4 zAHvl?`Nlwpau#A0uo^dC{O*CL?3MWoVAVs#nb}wV?nWCrs9!RefI>1$r^xa8vNg74 z$1V*p6L5H9_ikkxc+~U+ANZ98&=>C`>HjP>aq%HFiH~pj@r5UXrmLg*k7LN=cRJ_k zpKpJEnf;Hl1$jyl#+z=uc?A!xNGT|jgYxR&@7{U0b>SH$@Sk?-RC~|G@6rt&yrqW| z8S*tSQfF|2&^rH*vdk`wm#@I0glTy_@DxlGnG<4d`2@Zf`Co_-eRIWcoBy|kqb0P! zX%Uc(kze#u`QhBVWsj|YWtDAPwh>!t_h4)-&|(hd1QkxvK{ISUIUS0UawtmH{eG@( zy??%xC>H2RL7ZFD zsQ`;fI8M0$6YIwPb*?D+>SdCfIAPLFWDX|$X2Z@3%3y+bCXdkqiDNh~9H_;FxC`VN zUN{itKUp3SSOPwAWTVJLXy%_zoSioNE0aupigYIhKpQ`t75P@yGpQ&)Y}&d@=c-H^ zJ49RC5glEL*Tv)PGTh$9L;N>w*`$6kU_gKCjJC56TYWXzq~OVZheu7!omPr@LuV|o zZij5LckezMI%Kf=cMX(k^|k#>q z=-$)r`T1Se7h_2+ z&gUxB&}Gm;-;dpepL%R_8S6ZQ{1kC ztvj0^cokRbcEy12nIaxCC9dE!!t|>UgF>IPC)wcloUDV`$Us@A5|$7*4(?77GItJg zV8*yrXD+th?BBg#gCx9khRT2wu{G#JXK2Mj)M7xU!Xp?~M@xSX)h~q!K2b5V>5&Dt z;Wto}Kk?WhG#Tf!C31fiKnIVCLI@@>|=o zZHK-4!WnkfX(xjd`2>)Y1U?zzI?a?6)Z!xa8 z!5MjP5|%JR_#|i(M}X1tl^vrQKh6roM;rK=CuV_TXBEyLS^DTgtKPH=^2t6e;BZ?E zGtOvw=m2Rs-gC>-^v_;s6$7Ct!L2b{|3F0vm+27)FgX#WJTV66m~Oy?C@CL{tv6r29C8yTFOVtX7@3y8WBDB{2_*uzTK_Tk zY=5(PMhPaV#aLyYG1V>1PTGoQR(i!SltUezPY&*%O}X4=nx^*i0NcH>7rV3%KfJ{D&7*@kVnUKx|R-aT+( z1B;A5p0iT5yUOnT$=&wSEAw#mHk2E{!vn8RJ>_Kk_zfS%%~ZY7H<2gySI|lQ3Vnx% zN#FOtLw4`|4`GWOt~>`#N-+5ziJR~~_3;~QcLQ(@9yHLt@}kFiWRGDGAtxM|CyVtyqt?l5SG~AT@3TKf$|hV8T9lsvqi5xxQ(v9j%Da# zy?ghvtFOAk&OdvGEZfvP*k|CNT;CN6i&y6_vO9kCQ`@*{li-t|lvi}KtFFAvu6XxF zI&+ai8ZDl*NQQit6qneh&719xyMAV`FIuF7(5XzKvg7P`&9Lh}cr_;XWtf1o4S1QU z0OKv7)i{*;CqMshyEGz43 zpa1gzv3u{m&p!EykK4cfFxC-4e)2hYQl|Hy@8-p~OM ze9`R6C$j!6rm9vGiki4Rk+6zraG}CUd!_U&@gw9eq{qvHm=X^d1|KpjDkdth^@iG5 z9tuj_nvjR29@FuEvQ9<;s~6<~3(-=6$G55lx{JA{QIFz^G`wO5&4_TBeemtI&A)0KG{ z{83S|ed$(v_2E2@A{$xa>8C>lwDd>+IA)MZdF8+7@})^GY4rwtnt!qqZY@;Cr+u?Z9?#OmKlE!AAeR>W3uclM_*vqcfDZ@#t zI0?xE5DmB{7s9Lc7}d{C_`vB_hONRIA6Q@;fB!NpMYv$$@elkM>al@oo z>BNZyVo*+40&CDM3pZdg02u{abR?vb@`X@f)gvt?Y+ZU(*p#bIvA*N`6RH+Sh=BFu zqZ#W>K^~6Z)Muc(vaJ*hbt+`iB%!EN%B=gg?6M_aebAb)T^URCQlQZywGrp3l#l9f zlmGonm4$L#Y%>vBbJ+3Q-|63&0@x{+VG)D!!0cC-NTI;V{*0;PkzJp7rOT+45pV`f zPVagsN{X?F^u#mI+l`<4jLKIFdGVlbQ;U@1SPz*LE7<3qbGH5d7k9`4gcd(+=2T)- z;dZ@SzWd+y`+FZiyn0y%eD*V+vM+u1CeY`gtW$XyJo_IflbxZQ*|}@Ceddc_vDfFn zhW>~1Xt>f3`WMS81C~bEV(|TMeH~U7{dLn1X-UNg<(_}~*KgQ^4?m3Nh}C*5+)<_} zhLr`kDfEUy^4nkkcT9?>!ZHMBU%-N4K(FrhpWna3Zu!seKz`|lH2aZv_I@f|oC}mf z`Nmt^1`HZvKmOtOY!nW+=N3FEN=4vUKS&iO{ao4Yib?lFk3MeSzV!#T0aws-Yt*>0 zW9-h`zi0jX^+$a{;m7SU_@f0q^U68KH@laK z;%9FTmL-KMM|syWd+@Qx?O(q74cotOFRY8|kY`wYAb%yWGzJR0?jzUQKYZ>pvZ`Ud zv7EG&TeW7befslXwzW{`kk9cZFCc;iJrqZXciL&E*muA2H7lzq*H(Wv7~WJ?Rkg=H z_oaWfIdkVK|Ga68lV)1Au>7M(jk4QNPa}s9L)oD`WIf?OX{!e>y2wjtoyST4b1%-Z zPu~1FoV!tKKl*>S+4%8eq@?1u8>S=w03c7FxU=cc!nQU4WZP)OJf%;s3j4&Tzi9XT z_IEaI`Wbe|_r7JldiDf;aK;ek8yz|RQL)ywYgb#cbeUas?F~@k)!0Q)7XIkJzhyfs zX?+(KY`rL~u0!HK$O}ebbv5g`7wm68`$enhUSYrd(H%B?*kIiln^-%NTm*>6oq>LM z`FFw*Jsvy{2S3}Z*9BucU)7@LA?V+{{)yco1jHSNADl7fktoRg^*FYaGZp~FNQH?E z`p#sGiF8H@9aMG%kf8)BiW33h0LDox+bc67{EQ#m8^|yh002M$Nklx zE{&g==c6VpSd`!4BM&jp@Sx_D2W2%hyY)tsc8x_H zNB=bbg#Mw>(GsE_rhiZT%Y(Lc;btqwtw0T6*cIsTmwX#%l=P`UY~)|eh~$ME8iyw2 zvl3fo=KbtBR}=(NHsWBhZi9Q`R-3bMJs)qfL0Mv*glb`(hFU=w>IKS+&5ta!t&c9U z1GqJ(poD`ySA)RJh0*aJgaU_GE_9nP%!XWhs+AAxrFOu<5WF|UMKUVYvF)P&SxSb~E&okCm{j(fue{A%qOO8R@2IRsP3EE`q@11L# z?|)4eB_c7(b`Y6iSx0{KUDg-Z4@M=E!khiHg`ZIgNA!=s?h&BiL*FP%mVZd84ilp0 zc&YOP?yHMe+0Nb7sux`k%!617FkSzLi9ab1>X9^~;OBiCAk`=`8R1~cx?Z?^oo(K+ z6N4WuN(NiEa#$Ksp{IV)zWxvJ{sj7{MoAYwZvp?w<9AZW_L-+Y+I+tKd87CX>wn^N z;CHepAxj1eB>&SB86!TN?~)y2tUw%!BtFu_xQLYHbmpbs$j8V#5oZ?`JYLbQH?Sz_ z%dI!VG)WQKL-2p*D`x9R@iQ%R?fn`V3HfsW7zY1Pj*Ne{i<2;J$DQ}H=WX+A>#a)# z#%EefkTxVBh*N+}@*qY|5@@;Edu%@&aQsjk2y0DlujQ)trU#)Y!HqVZ5U#D-$dD$D zNI&Xt&xU2^2-F8M$^7sY>Sg1+wYKt^CAMeNPMmMz*Nb!P(YQqz$O{t;K5?XtI)9Rt z_bN~14{fJaxsSg5Y5FT)D0dC4VwYfha$^m)-e96d;ieM~0p@Lj6(a}OY7?bbr z_V`n?Y|%nm+|=5ni4*Ov+rNvsMVajfoc=w#+pTx})V}$hTdfZ)Hl|ED&U*FeA&UcA z(o$(K7Yduoos~8MRw;M>_jjxx&L*mZ?9A;!E+ernNmi#T`y?f#JMR23tYUs4r41EW zjRzZT^ypFcgYSI9`t|7}>za7zFXtIOx|i86esiCF70M2t@y9Jh7hib3(F$e>wxc}u z_!FQJluMY4{NJyA)vmns0$Anjg+i;?s;aB(>JNR~wr$&{x;pK&(`@Pq$Jx3Ko9xM_ zXUS>-DeXW0^`GpjD=x(uN!7C4Va7j&`hEU|7cr49v5_N3*g0pNg|jYd?TII!#x|U- zQpnu&iI3Tr{@16W9N{fRhI2#8?UrxdW#TPF zKAS#my4`-;f2lt<{eSk}0#K`B`~O~acb9YtNT`5-*a33w#6U$91Q9{$29Z=yR8$nZ zup1M*XrxiPyBpuncg^0#uT+L%YiTKxv~?WHFlgHg1TG`^FB&W!NQyn9dw!j5Mh=!Wmg&O^06@#LY* zrB1H`(xtC(JmhqrtWy z0^~S!1V)+pv+zh9M`n~JhNZi%fVIRu7}!Zcs4{l=M0Lb@qVOXwAx%oz6kmC%0fOpH zHU>}VFj0EJIocJMDe*lyAWEMcloT&6oA=cSzPPEedtfn2#9=*>d*GyOyzGhhI5Q9U zWAMQMsiL~-(Yf0}3gpwi@7hWcCx{A5Dnu@U&lz%&<%Q`X{AnPC3z$p@;zn;;y51&_ z``Q^D5^GgyMA&4*s(k&{r&=Z^NhOQ%AJ81Z7tO)xO%SCF6Ua`)Y?nUst!=^GpW>#M zC@hn>+C3CPgE*b--3r$nBWy*^s%Ax-92XcesvFUKP^o-$%3WLbP`)p0>#8lT+bqZj z_8fr00wd>!#?BN9b<>x_FaO4m_d7)ag!+)vY;u{?qy&V1PxWrvK6$w+(4{QY43V9~`S-VBmWn=SKpkh%g-Jq=HE?yvr?o+MUR@kcdX4(oo z-|#_b@hM#Ei_(_8`7|q3j{^%yDI%!iZKH22>e_JaWiznw~Z#So|`EpqK)2mvk6U(uQ)CoN4*-ZOId|xs^x`!mHnRqy2{X^m?3$1ygb+8xq zQ;#>?9k5HlQU{%cA8?71qIN7^Z&)B@5j2JtW%I}JuY_?b(F2kw*^9)6q$ecxto|AB zp!U!e#ZIbL2v7PJ9=-qVLvlpY4xfMhowaOp2be>RfT>#&QwmAy9d zG9>!MvoBlE{`WZDT&Pe%HiTQDSNB@mwymraj1nZ`u5J0bE>P^ z^y=E#En>B4> zeS6-8giH>n)1H3uH5)Q)gnR2cweiXJ;HV)keUuG}mub^ySj*P!VRjv`iWSP)mUSFN_2+T3m{kd%>n&9FiJz;_+n zM+aC*8{X8eBOCW$7;l+9T`k&45=AmJdMN#;`B$!OWxVyW<~M-5HFjH@R(31QncaKv zkMCDD6X6nNULEVX?%~WUIiRUqdS$w_#-NOQRY0s=UwxGtA$1T z3LhCfOCoF4y0tD{RFFQLH*X#v2U>r4NYKwpOE#1fW~&pv!1g&3h!GxCJ0VjHy8%N+ z+Vju9XfO z61Q8E&Ft5wIa8(r7UskCtF8KEuB~}@y6wdqPX^rn`NX2$qxKShAWX!NIEdI`_R@u| z)D zH|`4eX9eq0UAT5ET5lV%oWJht`L-X?U6o6OjPR)!NkF+spr<=vC9Y~<#m=Z2;ga%` zN65*uOZo|4lD{XO6Ay9pqyT>m@A}o1l7C9P_$j`Jm=u`+O@32mFSPZXObQQ;B4e(? zzsAtvg-*)FAH%zTb^Q_eS`>u3nW)~Z+^~tX>Fii}y~&kLyfPLRP4Yba!SSzBXf*w= z=ASh9Y4}g;fB*N!pU5dl4$B)E*$eY)%DNtA$}mr+{!N)fuM!%S(2upUceW-S@5kpkcNd^_zaZ zd)N)vT!uY|EndfBY;5m?A@Jasv8c%GfZ;IQ&TDd}ZDy~q2MLI_ccQK{V}><5_Z%BF zq`$3)i6GB0`PqzqFyRxsvqv9S{pr%_4!gGbrEbfi2>Xi>5015-{RUd)iWP0#nESa` zu$?epr2a8v_$YhonHStf$Z(h_XP(~JReDrsNkC-bzpkA~SMNjROMNw4mTa~db(ZT< zS<1rU)zAUGZQRo@*qhjqIKED8HsnZ@6h@+i_HsrVxXytj#$1>xt=hDQF|ppxKJ!$2 z9JP#fsAXtVFB>Id5=cenvx%Q08B^IF9euCmM*5{7`=hoUdtjU8Z493lL+WLu9ap`o zGiEYTzFgGf=60RzgO5H&DyW8y95L966)EIKbRU2A1snOm!$?u=v(cl5+ga343@!D{ zLbyQh(s!_pfAw{T{nE2fI1`CM1%qX#HE(ef{XC;xamhtCxKC$W4I@Ytgz!D#!->|m z(_LW}mn?f*2^B|Y=?cRyOY4tK)D*>5jA`H0o5bsSPD zJ7Ew=Erqz$KNC_(IXILJ^JGdN-phQXHWO_s`cq6e*$L9vkYtDOmpy7L!9M`auUv2L z(A{2p{S9jX2YcD~uvf`M|IR^w)U32aHisyHyg>%IVH(?kAyye=m8zLMPG

    =1N8qYmHH9#Q@0zQ(5~T3Egko!CWC?ns{Pz=D;Dx0dTUDF`==Xl{~c% zSDt(Jt1Q7wl+TCs?R|`&eAxGSnsDx;doR* zlJKvpXuf|G ziIQdOVf z^43EllPL6v?_nNSWJ%aM&M2b107~^{+KWHJDEZm#rLpul!%0#cCwFdwH@-Xp2-j(h{8eyR12 zZxWYZ#$m|9kAAW}NU>(bYmTq=THA$lHXL##thmQrR@X|@DeddK8{~{pew6wr`oG{s zaS~B^0D`IoEwd!GXr5Qr@EVR3C3w+CxJ?c%3riCzlX8|J<`|`RY~H_?!zVOFC=UMyfZh9RK@C zyOj;4PW$ts&hhxu&)cZc58GDkUpzi$w4HTELpJU~-6Lm?oc0bzVf)=Z80N`F8!>c{ zU4CIx7$w-XqpX{^Y_a(Zf3b3^N)#Of3Bf--a zrchSv)S)fv9#5P62!?270QbeGAGMMtO1UJDtM=0#S+U>p)9kr6fwo)?Gf5tebg(NMEOn1R z-p)rAN%ObHogd=+_`qfgc38gr?mK(?-3j!oZC?KT`3hJYjQfs$}+XiM8Atfi~f*z9p_G;{U_+k26ZVKAMqK!9%VDE(kyTYls}j z3D^m-HgDe} zFsUhA6TCKI?r8?DTH_PokQ3%li3M`4B0Z;X?>blv1+&&bWSSF zfGE9iq&Cjhj*V-Dq+U#7if$B>d2Eqg;NjlmxMyH;fh){^PA!=4zcUWdrk*f5% zz_Q|US@1nh)ZJ9as8yoJIBj^1e`6T&Jq}O4r+iC!O8%~A|8m%8dM0ByMhB~^B*u9R$3o>E2NZGpuu)R6H1m2`tK;~83H&JY zl(MBf#c*|3-}mbA*$+;5QA`Dj5w3Y&YvYXZKe*~kM%L-TM8OLRE=Ete^IXfpTB?np z=sr@#SPOY8Jvd(d(HvX;!ndAh%H~FGfSXb1gHp}v1X$W&`IS^&FJL&_5FRt5@hNsny7nH3&5#Y&w?|jI_L|FWJ7Y+N^xOTXF+&x}Rlq8hGnrSii@n>0DtXNbPm3I86B?(>u8=@(x? zWo4Ax0lBN=ZPxn67SwwOee+i&NkuWkFDMbC)g+qs1_`c*XyCSD3ULi z-O+QPy-0nEmnd!z48O;Yt69w@FZ$jy%*Ma+I{hVsz4F2n7@Mu=E^9Snv*Us0rSWgt z-9txsS?|Ahh@Elj$>1!#ee}shyB!GzF)6xt?&NjfF6ju$sdTeu&%tI&OXB{D#L30h zuU8k_j4cisf%Un?TZbPm?ep?ui1hkfZil(_nboUX&(&aZqB^8QTG8q`k&5Ya_fQP= zzRv;rE?12@2$O(_@jFqrDtgCp-jIwLnpVPxZV>8$fEcB!B0s12=Lxx!YjUI)Wyk2A?KqFyD& zST~pwCpE?{KG#S9zw~e zkIJtye1+?^eGJ$vc$sO8SYxPvsNYeuJyNDeDcDh?CNifys=tgOF`<1vaDx5(#XOh^ zNR*(SB4VI%*R!nD@nsI2`VM5~s0p7wOkQNWVCrN- zy5yubr~2uX@UDf$@h?EmoL0{<+rD_cZJxZ;)_*nEcB5vLK3f)FJSE=iRS|F{;2tDq zqJB(1asc)~3BN0-lf2xh1eLhBw&mjVM~k>RR5)Sek$8bC8{iR=`ciq4KOOp>h<*5n z!$T60Qyz#E!;kx>Y?N$3QbyycK}{q|n20qD!}vO+Jc*nRhjw^KB67+@5_~b^MC3}e zaK+jUz{_9-Aab<$sGr8h^8V=fcO|D?eO*=hJs>woc>(}Wg3{{zRL z+i;+cG8Qi{7}+^6$d4FvV4!O#Bu(hxA)bFzCdH&D$J1}TN1q%fYiY9An(l-km!0F?Egi*54-Svvb(0aKKgY%6# zHMKm&*{t!bc)E1xMAgA73=8Nf9oVe*)-^K08Nm)1r4B+xX6 z;yYg+!%TE3+un&OM10_>ocq%#$}}@6l{ig5s&`! z8||+(*B4&$p5FdNw^fn>i4w=7Hp1W_2VQTg_CunAQmVZAi?2hwP8^=4X;wlcqXj4&}s+uz8M=pycVIiZUr#HQnkb~B?bxZ5i_GV{VA(&)aP@BYzoHk^+>JjzZ#xgk;&FdxV-D;w~yzWyG&8NF=X`i<5D*ReNVdo{K_f_IyU zymR=fzZjmmbLX&czn^ODJ9Wkb&o=CEl((^CMzJx^2NNaO%-9dDCuwBuUqlCrnUWLc zNvrmq-A%95WFCEBlvS@$IT%F+H-%wD+}V4Oz3|d_r+tSG>Tg$Fc@Zi@NGy;?jDOW% z<^x*XdWTK;0Ev>?HEi^KNR$*TjB1438zU6}d`U|zUA7z*nT~kKS!sm|=C{!!hgsd) z)!Ah4wOo|vJ){=8bnollW@?~9GyeIZcXnosVA-$44I z@4zAMqImew0oLTKGpXA_JnW##)U~G@Z|&2wD-t#5P|n?IeZsQX?0E~U`E{-6lY8yr z^Up{9sHYo{72O#6N7P9GBx(Kpl<~qVuea}i_|Y!C=t8@@Z}*56p~t}VkiD4Jcid@j zz4@+HtXRQn9#_M17IS9N(#6Yd+47~7HM`qRseWt~7$rk3AL>R-^tN&1CTn`p<=z)0U83zn4VX&> z^5%0pIO{huuh7mKUd3edA#!xy1gawjEIvjn^2gdYyFiIFe*|KG_MzxVBp`&`{ zq*b!Sw$bR{J4k{|yeUi4!={UhT(~hZM!L2Jx>@&W6AV-AG zmgO695&SJK_OP-HQ-c|ud?SVc3%We&Xuy{|v8+`*yE^%!7fF2%V4=yyY|%%a^yIge z4d#XFzH{StJMpH*R^zHVsNB$YZlsJdlJCxyTQJf#$u|D@vx~i^XOWd18!bYrg9fi$ z0Pbg^%g9MVK~!BzUQ`Q76o@LmZ(X(77C`8%{%M&tXn&67E1BO3E!8WW9)TAopRMy& z*&1B^?wG$CX%g(DV9+f+1h1GoaEC%0Ir?}OXd0E99puzNgjGiRSAG~w`5Kh7ywyt* zhscC`6O$nHP5PV?@hQ6T5?us@F%iELpnh?*D=CBUm=vHi@vGnUqoCCA<1sU3mb~6z zbA@JX7^&WX;Lxs~@(4e1{+_R@loX)o@vGnUb7c4m7jESJU^CouNewv@O#PI;Bqpla;u3UeKCd|3av>%a2bVX3T`N563{>o`$0o(&e zUT=hH0q}WJKYuR##QYwEND9ze@9Ug4034)fgCdn%)&BtN<$BiGAIE$HgCm`t*0ZT) zFPOt-!YEk`qa+I(Ii(lHo$l%JQ7&+507+1rauM9<86 zeJY0U7dazGb<;P6*Kc#W<*$HYO4V6(P<+Jz{H~wh$G_}gC~uWRx`AK0M7C|SGCx_7zLTDG``1Di<2K+Qv_#1L1s}7*SnmNq_|{Bzs_tcFd+NUXP;Z+21v4(D(R9P z!t=zhzp);@2H2XlYu!lii%&jmtJZF!pXRgH9lF_jAAX3j-HdkO1?O3tmN&TVkxlg7 zVWURd7oUIS<*5Rr1oA+k&m0cCBxQfrCUC?Z0fr3hM-uX!%kk z=Q)VYh%{2yP95x=GftzAA!RpbzV+xcz*bRS8O(hKqq~(WmiNBDbm?-t=B768Md^}@ z@p{v*JL>Zo6)u>^`VSjzd?~XPLHiIsRv?n|EJU6^hK)c6{gZjSL&do zx1`6U=!ZSJjS^wpy9x*{C6M*ubv?;iNl)-70VS@2}Div{dNT=&+x_5^3pFdKQ($WDseLE&xPH$U6T zH>TR|^3iY%7tOj zWGgCNYDG~0KD=GS$LLqc_#66PDwxSHz*qlM-6qeGL`e`!m5q|@i07#efjEi_O5&u) zl+=)_MeP@FOQGuP{Y4_{z-!m2GZ~k64LsgW|)8a z_|qw2CJq%=uGwIVSFH7Qp+f1RR-sf;HZJmH;e)}ag~U%hm&W(RheM~?qzK86(fquL z&9scU>MF|VvD^d4G)GdFhd;)7#rLEbse_WQD9isG>3zjh1>;a{j|)e3j5qfCHHKje5l?Os#t`b8W|m_os;rS^%%pA z?|f9dsazs+r+0YC_Mu{+gCQ@g=-D0nEE|Sxs}H;Y+_E;3<}e0EDj>h{62oxx_g)5_~oqP4Sci;JdWLEj{2+XFvFrLIX(72J*R%ZI^+VvY?(Da01Fp0KAiU)X7dl%C~j2#&&mTJe# z7*;J_jCqRu(@XW{b{Hi{lr(ZTw}X(j$jP^Z&cu;KNoS0*K8Ko6f&BTgQ8E~t93dlTHytqhzFy}K`la`D1Lb>DpFL+jG3 zmoqB{^zLE*YI;t>wuz)P=FFduvDcfC*4S+qG&>J(LVY6qX&lE`=N^r}U9{nKH?_0r z)2G|5t#7h!cijP#i;aGgTAUcyl1>s%v^NxuDT#_K;HOQ;?)J*7Z`dh#^?CO3hcWEB zlX~pNc1vfQgv80QFhXSfwp7XD-k%DfS~X?H9J>`J+A2I0)jUqNV+LDZBudn78<6s7 zdj92}N6lK*Z9Mpvs@7f(1`8r}IC|`}*lu~yrIyY-{WKU8Lu@Pik^@K(6(aAR_l&>` z&3N#?&z>1O#!fupc;fDZVYtHG18>^6!7fH}W*Cw-Yd37h^I2XPB@ZA`@|>#(-QB;J zwYcIUTL*ognKeohKR31QXrF#M5w)or*i#v7B}x=^6*Um*>NmI3Z(e`>P1JMF!XwlW z_R;idaHR3+m(U(&pTs?Gi7*HVj^g|y)_(@H`tc!`g1nX~;YSGaj~<@%F?y7Z5-n8f zC;}04TETTlrokT~MCj%N|2)1&iqNhVBhet9p_j~y?Vm_Nd~HhgA%qjA(*R}~ZOW(5 zTkK2#6o=v{dYIZmc~bvIl}HXeLXdy<@T8B4$Haqgwlz7+71fuMuu;NhJ_}5R z#$uF2=uf@arKC+1;Rr$hF8nxY;tos^0-6{;AMz1y@#a`IRBv?Ja`4yvc2bAaoUyT= z2~8YMzcc|;2Tg3coMECi1d;0(tg`tZ&aw?ZuW=$rlaEH0O9PSMAd9Y=Fn^_DR{f$n zmZJckhsa|;-2b1(Pq8`g&NNA=)Ngf~)xM!Fld@c#F(gIhM6etz?YS38rPc4uwAB-4 zI#VPAQ`%xt-1c6li;yRYSNS@?gglC)`wtDP^fjspvYq@wXH>DaA=Vd8SD@ zCms_IzYAY^2`@T%+J*{_6Dr;YDx3O1N?wW>J`v^*!u8?fU!;zjP8g5t;koFCOOi8x z<@}QdKMntB{qO(I_+x2jiSFUFrIA9e@7qjv(O2h!x1|?}0OZ6OPq~zb^ zm{ItVf;4yYw;8WD-@_=;bSTmM<0_YC&XZb3w0;uqQ)P5ljz`E&!O?YuuSW>-$R3{b zkw{$QLnsl;)mlkVZG)(u^LU<56)fCHuZAI#v?Juq5?-#w@q z*Kd286|7J!TKg%AcDP9pVp#Fr^W;adj&w;%M)c2AF-ka%7L6miLTaf8*evEKTg0mL zZsyD(Epq9^g^Y7cJSH9zB}oZME?rat)m4NP_s>zi(I!qdAVe=aZ5nMt4=b-{nv`?m zG4bH^uB5FNELviZKJlz?>aSwsR=-Xyr^|z3%AMBi@)ovm;V)LUTp9cP-B%IlKrI&> zJAATpTQ=Zj>9Un}8a5vU;nr3++MTzxvT1V{qq>V47iF$ou9TH5oYw}9cnC(xGcLjX z7^*j?Hg4dO2+B|PFuwifN4v9oZ(ED1Mb9oBt>txBbIOhs2W>6Jha`#J<^@cb`yYD3 zM#1dRIIdi&qK(0wa9)@OqA_I~Fv*dH;hCVU*Oujs~hXMGAmZm>l5P`z-Z4 z2%Rb;v67s4{Hf<`(v)c!4_!(*bzqEIHQ8=Meds!jqxOVJw3Cgy%B4L)>-Klrr=Lx< z1~5uwlvHkRC6TX#&%(gJ3pJtVkSHlopdj`?hGNU478Oo!T`_R`Ch9C%kbHRMxyPxm z7$w+@p*A@v%j+;v`k{IwgQP=;46yUgK0`bqd+UP!EHT&wTud#1Z*Pek*0_3GN_;rCcx=3-q^y*ck98K~W9Ex>oz zPVHPhQ+q-gR-H5jh7L>?wSl}SU3c|mjyH3B;M~9L{3+1&=*yec_ z?@5)bRJKu+wF-<&l}`q8Wn!RXmtOYfTklxYCQaz;Lurpa-oG_3d5;v5{UHVadXM{` z`hV1mJxpUx{lJ$6?*FgR-}C{t%--45q*Q?as(lNIs^rHtV#S6v~;l>5=bJn~^bVd7Lr5l38%5{VZk?4{7*g@5THCg+=+ z&3_9&1*1_;AWe7^qQ4scvCW?_+p@wa(flmVboGmB+KDY2`C@U91(VT3TZ&;T!YsOk zFy4d22fC3=)Qb>(U4miQJzMu!CXB*L3gw{K<-}F}b+VHc!>Y&Bt6*hMtxO!GLcs3E zMH_75zz?iUy>eFT<_2(3#N~PH+YyW!o?7Z1EFc*{)?9yg=z8Ufi=x zD3;evBA$DZP{>>$mlZp&rWHJ?JVX<=Fi_*!v1py`UW;BH;=t(<^%Y5}g}|j5SoM5h zQ$JtSsQdv<_~0x9ClR?S7qz0z&><{a(1|`3!0oxPSj6cRMy>gYThC*8sGd@;zrO!D zG8|xXSwaa!P-|JAibTnJY?KHQ4P~Pwk|+(z-{cs{&v9J(5Jgb` zm-hqd7;7%j_1}zt%{%|g`6ms28vfJz-~Vp?|GUSZHdPYOUyZ$k#Vgi1Fj{L`OXz{M zvsT;eH>M#)vWf!^=5;o5ZkU~QHa<|uk~fQ;JM>~yX>z05GQ}2uyTCm`XkF4KdEd@` zR=efN7?&?$t7a{?&-%XahZG{__wUAqDkty%8hMEotXj-A%~)nXKk%_LHhdkYEbta= z|F%7rr*;{u()k?JdRg1q>}p*L*YD%sz4DM|;lw#M|GgQmtM8bko_n_LvXk1L6>O9^ z{SfCHWuI6n_5S<#cUm{fXU3f`xk34KZ2p}%1KvUEoDHvnRr_4vj5%r}bpOVc3^VyT zP7>8$WdkRI|K%iXl&~q!>JlZ`(ewO6+89)vQ+$teJMH}l__Dz6fI~&M?C^#W}Hh zxd$#<7}XmLgSzb>r{~C5S(839d!7vGGr=%f<*Gw6FgpOFtFvd%!N_MxY^;p5GNno* zt$`#jmB>V!w`q@!l23w-lKY3P+^^v*WP;f#)9ZscPf^->>0Gr~kmA&eRzJqvV{k&Y&$0+Pf2EqokLs z-t@wY&?U{9x>|_nf2r)ung^rgx|ZO7zg>dL%HX~|nDfLafq_Wgp1#y}6|M z4fg%_Kf=_w#0K{1PT92&0hbg;dCr};z(zjwsLh!>7lW?5VWg;>+T5qJ%*^AGUXksR z7oK{IcKOxjA=PsGT|I0*>25$pYe>Itwgy+znW(F@7IM>1+I8rH9hSH3r1~e}Eocy` zH$gpcNkC9znwM)NtV+ZH*|)_QpUm6H@1dRveEG}bO35(8&DHs4UmnJ$Fa|o z4=+Uz;2Ejz3ANpjuuLBmb@i!1?xCnk6o%7}0YgDDm)b!9FySQtO07lS(@DgTEYrG1t%L_{olpx$2TN_hLv()VZNP z1VEusbtcJQ!Kb*lDY)=L`$QokNSE6KnJ~lF&R!)>hZ9Hsf%efD6IWbD&+=d=q+*lm z?uJ<3_ri$Sy}&GVItJxq)Xq{d^pEey)(u3tjAa@#$Lk!=8Fb1 zah%ZsOi=)t^5wLBxUH5tkaI>MK-B*<(fMQx9B&Hhs0mYpPSP;KS5{6a@*Y>(ikyRb zO|if!R{k`mFXYM~3?)h$(4w!0T}tDhg+_r$#E8E5EfI^a$>EBe{4J6A-+oA><{=sW zi_%c7ZN#S{RmeIasqsZ`|pka zLEBuuExdrk`-&5=yr*cl12PE zWuE!-=6|g>+T3XK_e10!xLBlmBceZs8Rnw!8&8LH#Z>cum|)}anuz6nHq|&*!lTUp zG4n%&_Sg4)Vn2T}&t3g$LG(?HXf~BOf-Tp(vR!uJqs=WRp5bP{$*K2O^DronWG8X7 zS(NHc?Q0ua@#BuM)pM5H`**zNp7qknh8YXRE?nB5JLm!{QNNsRp1s0;9{Ew?(86)R z4sIXJjC|F~SmmBg5{6&@Nd0vRtCQ$OUoEh??@ac(h=>%ev3JWZtJn4nD}(`guRme@ zJBZj0S^xk*07*naRO*$$UW9{`sL>aN)0srO&gl6nYQ`IU;~7ht9M}V+q*OtxCW#W+ zPmB7%Q2`$E;b`y^3=Qb!FHjE`qeS#_W9Vfac=>MEIWmbH4$1k*uTB$DNzre57KJJ5 zO^yw;@p9w-N=lqwcaN~g$326)+A{V!UQ-HVEHqGJLbf)+lC(*OM9C^wy%7_|Rd3qe zO#ff#3`6z#%H@h$35=$WeB?rRmA@{+Wy7sd{+xDw zn@+BJ<2FiQloTlvxd#@3?a`Hqu0Y!=OpY7!VGr<`@o!o0es?ZTj7vmST0bGsJW zAgj!|@JMvaZJk_V<;1!tvf&?y=M=o^(C;)33L_2Pu{&OG#*ar`qZAS)gRN$D_Ay9S z^dCGN&o-|*oj?BhC$Xhb!PQL!Uy=@DUyUDXx!i?a;{$k9NPS zqkFxPbcLrAqh!upyR!L>pg6r<1>>V{_f9SyAtT4ai{i!jm*h)+m^oKmcZ+@b^*46b z>8IO>A^lM~$?SA~Mi@aGV2sI4^;{%X)TN6QDPo1_YhrXudSvz5HP{4MW3^E!dhW?F z^!{`eqD-HTm+PdPBbu zqonBtS0FL6!*06qdh2vsYj^FdE+eC?Km9ZdFFoy$+SmoduCq12{GwpnMc=9a1Y8Ms zN9{{E8S}t!7$uDvj{Du~%{39DB$g;y7pdM1kVFaYk@I52^#Kg-o`ky2R&4A9FH17W zD@MulsJ=;}WDtgl%ee<283JCnVZGhfsRxE_zjx`G(L?)tT{B^XSqRry)+Fw|>=68W zTZAv6VG`eYB|J*47k`V_D9F_JAU*Zb^sz+RQrAivZ zgeRroaTA+imOhT=N0ml zXp%3;#GJQig}dsK-s169%P}Fax#cf#bt52$+*HcAuKwxJqVoP*_4mY{3FYRl4&kad zh&AqGW1b!NV>0F%P(*Fw%%XC5(&RD`kP212$4Nrmq!jU^-;;2a$Qwopn^G}1=Dao2 zzJKsD)2T%!2oy1vv`EXEyWI93izLQz*PY;t8mXKZ&_belJ-`PD4?2tZ6<1i!2`6az zv?aC>Z#6q`D<#oUEpRa*A+E(im@;C#$h%I(bE>(diHa8!1io;Pk0w%GGRU@k%2M}W zlM6c_mCmh^NV9X*7F&gPoDJVBq#V=>cq*$9uZht63ipsV4=OL^7d}-F#c`ra3ys!f z%BBfd6J+)>g{;8oRV;svl9st3-dHG$hD@kyz(HtVEp(x*0v#giemMSBq7Z@GQLe*c z5b$TUe=>aaKeeM&Z#GJzgy)8}Dq}>8Ns*6itK@ zf7DQp-T%e>lLlW6r(Ue~)BF%$Axt3}8L2xpaiWOcFQX)B?Vr~F|NlPzwf>3FUbPO1 zlI2L0@K+JFoASpLf!<(qs6{4mX$&O4Cq5(|4vTTpj~hxw@?tTz#J(8vAvFw#BU%~b89*NGd{e$Zos~YVl9fivvt+$v+`ie| zx2M~pFENzK`YrE4LfJkfD2{7h-%247vJ0t{na})a^WLAq`fS)I$&81+UDmiqQ!CfF zylq9rXZ|o$bL0hwa)ga_V3g#kevDNio(?^n$>NLY(c2&y|4uhR8!Z0eXPfiJF+-ZUo=fv>!o@#7kdR5bA=`hHcG$~M2=M1V~qLy9K8fwC1EUn zOsan}EuqtTseicJV6F90TX*j_#GZWS1*^cO?Va(@xFJ+$Qi%p6FB!U({@A7dYWz{Dt#`hhZ&7n9fCe%}TT z8ji8ejd*vt2ahUE{JR)%qJecNC(kjHCQYGD)7cF$7P@q3XIoKO(P3SX0;gb#PrEz) z$9R+Eb5`i5x$_p=SJP%l{(uWH3MBn;>6O>9kH~B{A(hh!Pc2)J zqLZql&u`Q>NG3^23F!{nwnx7~7%hDjJ0c}*++(Ay7|f6zFe-{*VD*7VpR(a2Mp;&* zT+T;&r3I=k+3_$V8^2PY&O!C&ipwsu;e-0P z@mn#+MgPlK@ogQu+XSRA>YZ5EZMQh1gxYUFeNJ9)#0Y7HTj)F5--0(FNk1XI0`p-e zj1n=_WWcs}_q$+}T$-@aA_m+0AAM>)QES?cM9P>4M%Za5H*%)cl4Ypgw7dmluAA)A zi_f-^QBtK6j1t+T3in-`bKfvK7PI;b2)~S#1hhQsF;&|Lq^uce(xv^k0He zl2moCoRHzOPIx=)_ptb>V*F~GBnAgMLmr@23lr}`nt}tC10wbApB?!jOyD*BC_Dw0 zA}Vc|2pA^3DguF?ukIe`K~ZElX*+!5t4StTuAIyWiXO%F1d5qxdNNQRQJ&%Lu;0Vt zr;0(^xFFo)MaU&erp#rcz&I2(N>0Z0n!xx)xl@)QLjFYf5rh~W;jS7-FlN6p)n>dp z)pl*z;ckMRn9$^naaT>UIT4Gkf9ome0+;rzWVki<7DG1r=a&{*F+;?p5{G&pQX;Z{ zvIKEjNoHhX@o@c1@UKNigp5>svgFKa6`R$v;r=W>sj{F1^k3d6_g=35i811^>A)zRDXq~T*64y z5&a&0IV?C15Dx#k{+B3M3@*M4D$gqWBqT~UN}>e3G^kw}gJ|L&bET?R2-5Fiy9K1*_L@j*Ph)HcE=PjS^@2it6{dD`|R61RP;L4HGkwk9>tWQQv|@ zNs2L7soosJyw3dT6R7YWU4J_Kq`UzkCWusG7JNM0z90RCuPcJ=G#u-;=!)!xa={oW zXI0LtX8FqH=c{N25d`UM-n%nx5$Y`&vU0l3L5}q0_w3$l$6kDbm1|H@IeGk5GnU(= z$G)|VKd)v3chFAl1Eb{Rina}=!u-MSIjtdYAyWHQdG>ADi5EIlZ@M*cdPcBx*b_@p zkSRGIef`zpk)$7%;E`uGj1t*BQG6Y=NtO5b7AITLTBV%!(Q+S_OyMR*xFB%Fukk$* zkQ%*w1GDqUNn^>a^)@+hXE7;{!<1TT$N1M~5g&uu*b4HcE8edP^|o z`r}VXlptB9GFB~D!U`40Zy$WjNjx4ywrt*F@-)(=LmLcQZu5Rv1kWIShumjRKl>5~ zH2ZAy{ll=Cae9!h0N**GRlBp9bOW=r^J%A^!pZyHevl{_LWRl|_!~h2$ony2a#33Z zm507VM%t54KkxB|4D4%{UeMImZ`*EcH<3bCYT=1r+E6w#oSx)e%po_pDrODQVq8m7 zs}PI~H|C0sk~-LU8I43qp+ZP}?Az=8(SdokjH|$fA;0~W7v_OfO&)w?Eb#D9b7y;) z4p%#4NK6zlt8`FR1htHIoqJ)(^KH{c{{h_GHf~TCJ1(0nM-JKvCP>RR9gvdw4wai) zNR$k*0>sNq`CoYD4ZCO9eU8uG-8fmVWB-| z%$#LcU4Ii5O@~d4vu*US0T|}p;`_8h1@hR~r(d>#Lx#I?;clHfB2{uR2j4p^6XjMf zjQghWud$O2+W!r_2=(qi(BEHu@oB4Ez8w3&?eq%_oemyl4?g^e)u>V3o_OQ|D~?3Y zW*BI)t(b+w;S+WYT+U=15Ia4$1^_Uv&95E;3hJb8*$tA4CK`rv)2JQc)?(l+;Q zGY9pc*6liB?_`D5B;SYb8|F4_#B|#zW3EV)$jEFfZ0dBl^=2eSf~r$ym_I+wnqxO0 zQL+<8O79+>y?*ZTh)yPE&Id3`di3s(#K#V2l$?r0$$^9WkgQsPEf#sb*#M*DLKr2u z2X@t)yfy$s#*aNdjy8lbglDAsk|=>u!q&%%un+Fs_Z}olp2glyWmglDM2Xb0gzpbf z^|`6_?N%5g!S@axX!YxzfZC84aq9QroB!&Q|EK702C4ToAFy45Lp&9@_*gdg+LTnGyOrU>?=J|Q3r}fkXYO3@JgJ)m-<21@CB=YZ` zz1|}bF?{A%%f;zKW;kMg%D@Dq%cI8eJtX`j2mPn@kE-YydKoE`Ci*XN54@L&OO~M< z!6@;CPW6v)tOS2H{G;XRO)sVa+|aF9q@Vow*AOc6+-8T2tvUXf#C1X=qNBnY)$F*d zYg_K3xV=hr8x>yF2m?on^iA$|W7op#=B=@XpJ43u7mU4%D+xSJ)@nwbdTEh9fCc;_ zNNtoqql)Fh{oUG`%Zb0%79;LTG-sRyf`RgqMiaIeDuvNqu6RaO%Y)vo;tBSW7r(a+ zpUxvs^rEqY;zXmcsxqs5#M5Mbkc^K{@*7!~wW2X_y=1^2+`oI;sZXB}`X z!cN)O!#`oi$9FbK2RJ0jg)#MtXI8cH=T^4ts1N889MK7S5<_GDdo!^EGRq}U^-=S` zBwMRpSjWm@lwBK0E&kecYdp<;W4itL*f(}OcFn4uU(E?(pG{m}Z8z?l_GCoQ&Ful-2gp1{-Zfl*SY zc>^nsM?c?%kf-OBoXH`13WhF^a!5Rd$89%f*n76^rxm_|6Ro06*FkK^WiMICDtB#a znNVAb+vl)s4hcBqRbW!Xk0L0)&ta5kqb_5vC)Y+*T^sfYE9zrMx(>@fDL<|2qWdO% z{2A_n?{a3<2^b=M81Ej%*yKwJVDNnpxzSN=?oU7URC|2%P~YH-F%q{;EM2k$SE}-Q zqZ-_LGmH|c-oPl?!Z-l?c5J1R(EHi&jIz}(ZGOF2Qzxu?-kJ z95sxMR;pA9w>wg{Oeu_PVk|dDHq=D4>6ho7gI1>WF`UZZV}(#bku4APEp7gVs~kuo zsC-WU5+=!z4A{=twq*;9fiCv_kCR|19Pf3_$7#I`)(0u+uqGaWIgtx^?@svCKKuMj zw*_)-^DC`R&1$F&$+a#_J>eh3?_(Uvn5ra7-h)wc{P8deX#X z|K8XeDdUW&;iDe1mtTI>?X@&(dXC*UxW8?~%!0h<{IX~`{^}c-x;nNh?t!rjk~be7FsZ}Fji}yS zD6cm=?dF!)D51Z~$gSobxhno?&K$e(=Jw7A>D{xlU3u9h*eIbd^IeixAAS6}b?wuS z@7O3oqU3bE@$92rmo0};asxI>Hg2?waSuFHj1n02GS)f}_rPPvJ!yH6C>f1JNnQHN zb{H)>B$kV3F-o3%`ZqmFbKI6_b~{bI~jj1nU7PXm@|-V*`Ph13)lU^xjH zeO3Ih8CBN4BaX|X5kVS#4TD25lt~B=Hzq1UB6AH+htJxoupDfdv(hPjdQt`qvH(ed z;w#ScNbiS!BhC?m;=u5UUmwBYO{zCvVJJ(3JX_YxNR(9NJN2i$0Xv6gnEamn9F^|~ zK}Uuk*C$RVm;sl9FC(TpVNu$}6MwcDFHW-cFex%-gUijN5H3XjFvgl2wU-)~o?w-5 z&8Je!o=IR}kRiDC(-)X5Dw-G&TIf`{eVhy}`+kxA@(KE{LaA`cV60BS1UEa3nSzZgSUygLvLZ>M%LzEDt}+T&q0>o&`1o&be zN{n?U9P3qhjG$$$00c$AGh;aJe_l@2jJgSKR71ae`-E@t_z%@0VpP??d;Up-uP&B` z|Fr%Wrka2H_>;+g_2gfSYe*SiF#u043k6RHRVAQTk^V){GR+A zQs^XFYh!GqM02BvWEpdn>Pk4YCcoq?mGn=7pO}I}aU%O4>1^2#i`<*uuWSP4`C|`a z@)gglW+!(!%VUa0&|!&>4MLnTzdSjAIOQ>MPc zLd>{UeK##$&ju2=sg;Y_u2q}ur#^4^+Ah3^Adwn_Hnb8O&r&>}mBmPX_TqV+=5of4 z))%5G=YXbUWCfgubugW+n7Yhny*i2Vimp*SsVMKZYL}j9WhL< z4N?}Fq2n6YL83%75OkC3mFfzhQUwW;UvafTDgs$EvViI zZ`CT6cKa$hvS+c8qaU*e$26hfUrQZ_L?toq6E!4215Wfz@)p1W3+@nK2EE2ap);!KVmbi@)q^;#{IGO%Yhm`bsVWd#{4jyUGJol2* zzkU1k2ri0|qR2@9kM|#L{|_dRm?l?Ve>0K}KRI)vB#fjYFk`k7f9cZYZ1Q)pq0a8= zB`4KA9>c4sOH+5L&h#2E2>T_kIRhy#22M*K!(7C8`FZhT+d+CsXGq$l<#ktL=LHoe zhDUyk#g2XI1?xX(h{KcWOqnvJVK|}gwq%)YLlR;KOpfMPTxR`xc6FOGN*DSgEuo?4 zH<>rU^)Wb8f9u$_mra;}DoouI?D?l2C7bpfcTOZbKKabLbnWZkG{49gZhrd3`t|GW=Y_w(V9N=U zXpdE`S``M?AiM(Q^SNy^OtWT}Hbgz^BhKsgQck3fs=wyYLfC5THgK3djhdT0INdvRpxY?fi&~$&CyBu% z^{gwexWxMR=;rQ`L6-N!|Eu)3GfI52RnPNFQ-H4MU9Tc|c|YWfzsFOeQpA&_o;IO7 z8qvv7xI(!ECuGN+2fcFB6zoft!}6Gl9K|I9om!{ex;p_Ll}Aewp&nOd3Co>Ah^Od0 ziHLDVNfwNKdW@)UO2+k9)Ia1Lex0BJq%%sSdc$NWeK=RWQHkih@dO-7taMQsy(*zT z`qdSeJ|#4DNQjm6>k0Wuv~lIw(WQy?a@GSmj5r*03Tb>dHtXA_AEL#EG=K_B> z#?R)yGu>8CUk2VGAaMn#1tc+z7OGIh zrAlOc*4_5eKb(+Z(GV_IeKOZpqx-yj)kZ(5QXl$tH)=rna#+qvMXUfKyZP#rVN!;n zqZGgsg_$4c7tW4pI0nh%r_63XMEZ6pLigo-2c=0UvsDE zrXW!w$;)5yc#z*}R4N@#xMO%e{P>CAQ$YLEe+oY|W89dcvtS@_s=Z?RQv3Yw33zB> zEoHN_mqW8MFcMDbd5$x7#AMKl9v8<|Z6#6i{!IJ%GbBpfn5(|C@j;@b+&NXP%Gou& zPAafAaw@Y(vDs(4ou%UUoO1%QF!hg?de3aHkH zTwbr=fv+Tv*Mk^yo#P%xWB~INBua#eV1FRw8$$@tl3^x4hq$Vf;yXTs*Ixatz(i!lR}MA>Nl=V?Eu=u&%HymYD6Z7ema z1J=1iTRh3M!|M&Mr#bmoykjeta<{5<81FNKPzQM(n;W}0jFI}gvP(xY)Dpe@!I1Nk>FQ$-6*k0_8UEkVXfBQY`PMly*jTwdN4PI~P+fw~g&y4$ly!NP$Q>ISG zmPKo%4c6mn=1hA8gP_8#Xur@^LvI(U`zv+Lk`?f)TBqA@!_!JL+r{QyDk~a`vKgW_=sj?-eFCFMOrU)rRizRgC;JzdUUHFj>)4+1 z?hCLfuJ#VT+2ScRr|g13 zQn6wM>?LKf$y29Uqx$vj-Pc~AZLw=}Fav5ZS!@L9oq;d=UvpX_qH^hjc4$UXh+ z+UA$Lbce=8PHdFCGvPzKy<=AxQrqq6Cmyx4&p5^B+MjXveCZX}xYwrUS6*fJ59?=Z z+&yr9>oIV+J@n8cxF{~*Wodx=%w|-NgzKWH*R<>0#~vR0gw?E8)y6(F$_?-G$-e&P zJiTRDn{BW)io3hJyGxPa?!}?F7B7@iEV#Qq%deJ5BZwLDcl3${W zRT(_vREZEj+cE8NOmcCl>weChNms}X7@kj-XDLQOmxrSorr|L(gk?>cS7eibcAues}Q69iXH}C8hh1427^j{z9 zejJVV#a2FysFe-EoaBi<+03y#{{TroYjc3BEO4~FR~?bTNH>2H0numUU^j4eb%|yH zVbAsnvoCJ`0CdlUiX^RC3--?VZ`E}QY zLip{2fWx?5XKKWf01QJK5vtfm4wm-tiFw*EgAaC(`?V`Ged>~-%*41VY&_kA}^LWo7@ZKfy zHeGEn(otf~JBC323%JTr!L+4o#?bbEK{^Q_Nfk`nVd*u~I% zlilufAsUoZ=;ZymTE;>k*fDQkS!Kigk;EwFBCVcrZ--g;M?>E!pBmHbkTUD~7;W%P zqQIY5)3ePV4kE5U47zIA+m=2>mVhb9>BPo|;h5zAo%zd@_>Cm*xir5rr~5u|Bi^UwIcneUK07aA9>OFv@+>_p~Drl=^8^*5_i^D*bI9C*PmyKrpy;$;7GVpX$H2!u&RPrur>ia(5xZIt0qfA7iyBSqp;8BLCHsWc&dRArip&^%|4X( zngr^9@;`SOu$hf9$43}b@A3@aEkx4>DFIhqNQGT&nBz(BeAkuUw*CJgZnS+y{U|3) z)!$8SChpWto09N^(poR6qAM)^M?;d7*mKhvnEk!?o4GAi%)Tp=&2WmGcdZeSj(fb| z_ThOIoNd`$@u$H49}fA+W!dQ*b8-jNvK+d=&lhNnj=HztQ}na`S0&2d9_7Ia%L6iN z4At0D&o#FB2%>~lXRK-$kk@#m3F5AXHU{<;Jj(36jFB|E5nH;nv+WULWO6KP(6TZ% z7)=Yy5>ZToWT=@xX1Ju%Tq^WWX=Mhd+7-P2vSLDptCT%ja(MDBf0BZf{iyl(`(LUj zz#;P4$q3dr?J&vykqT~~Tyyur^*3GA>#GzRs~92&XVbg>!^KFE zrz#IOR;78S)8f%#XLOd1El(xPEZz$aL9-RtwrO=yn?xzh1?oY!EEa?#`pjLqqj=Ld z-$ed>LpV=a*Ux*bcdlz+Fn6aI6d9DV{3EoqVSl^6crCQ(c(F4MVWnn|8ES@PMuDAi zd!_lmM*Y~6Nzg|VDM1_h@p&Y8R_uUSg!6k2M#4x2HGhSIFRu}k^TqdPCZ<_FH-ED+ zhU6!?G5&XvqkndW`m=?TC=qY}_0#m}R6J`dmDn9Pi2Yfz6jA#$u7nO7y2xfHPX6ex z;#@i>4;X7i1DL+GD#fAYHd?X!w@^}BWHvE-wr6dD)F(U8jGwi~`r}eVuIvv{bAT)V z|6CTZ9Rj63;GHABZ5gU4d|=^?sHAjV(jNxaK}&pwj?(dxL;Yjr-vbx(_D|p4zH;V# z1KEw-Mi-{A?X^15lY4Vh&x`BAEIpnJ|5OG(T=s|8*UD+_83~1J^{>pBxfMPH&?fWE zVa}Pu(C|jvv&=?86d}xe9As=4*AQIg&EH>uV?S44lPlyc8+v2ncwShT&kY(VNDFt% ziPM=I-?5%#HB1oil{^GLoiDmxqTVJD;&^Y4Ggh@e`BKQ&0KB_v=OXd;k5w&ue>63PbANmMYeN^v&|<3wzoR=)|(Q&9GJ=Kcd_V z;`vS)K!$26JE)E7MW$&n6*m+2Ly|W>RtBe0f$OdaH33rS*hHj%G?UEcdmMc8(QLo! ze%F*k+4XTke=YiC#$rFzG?oCn6FoOLE?&VmQA@4eH{itw)DXkbR_P^snCQzXH4?n6 zYYaC@NyVS?UMAi*=C3TerLi9fg};$$@;8nW2va1+@{f*W-!8->zt~P7?lCtSE7%(%`@%cbSfP$ZDwW8O9*x34kzKZkPD#DxY`S=!JN)c zj1hxg zo2U#NYca_)@=Cvu+6HC-i}X3YzVlEps?`G?>#0TT8xfo_A~=w6p#g$JdWRJ#gKNeS zf=Pq$wJi0+@^onJXcZjBcQFV(dt3zOJ31M5`6qqg+&8yg*ENiQD`g01SfFBJ?VC;y@PZm?9F zY;TW=li4qH&HO6qKRl_Y!yvEkYA+f3MTRt*g8X|>VuHSYOo&*8UC%RL{I2Sd7ED6p0XLK5gRU8V9eKv{dQ%;)W>qOC|MlSNJErGj-Y64Ys{H(IqzP zcAKv{-EMU2O4m5qrECDHKot=wni|W|J!K)qXbenRl=|ql@%t-j8*gj#Ol?v$VI%}+ z=fpfjgfT?&u*-<;dU&GKaf=K-@45Q3onJV9(BcjpFr1Hr={PGEmeQTkMzcq5{RS+^CR)UTCYUIZOx+mFBuBZGdgC)OQ(xas&{?eONf_(#Te1*=bdVZ^as zF&U(QYTBWHbKvv?*6_po*(Z6SS?AJ@tKJ#{j*+0l{hNjk_l7aUElwD$;NZ+OJbHM=YSm4fh5=W^xko!xj(cB=}ssf1l`QQ{wB1E^ERN1an0cv zsT|ij%)!}l&L?`@b!^{;+3eR){({6^nP{(GJPdMNIoknSGmSZ)3J_{ zm-{%dJZ)^pc@-@s+M!DMN}uFXH!UV&ZhbKGwe%rk@Tp_WIS+%zG|sZvIW)xbwM4Ir z--zaD_Xzd9=Q%d5^IDT<_cdNWM~qSaa8=@9V&0B45;xX`(D*6rTv1)hs8$!-O6dgu^T0hw(v<7p9@y&UXrtVA1BhkP%6O5C`8*q z>l*d6%IWaYbxzC4h-hX5hdK)sXN*&KlxaDF7tG?*XYK(px8N_{k_9Bu#j|t@ zMdF}Edwni6ym)W$4edyQi6R**lyz?T`6SN&CKjQa4D^@`R@E*kwu*SD)QO^fK|*w! zvE?9I^-7@EPKBc7J8z z96b8E#ht8&G*{=@fN5(bTig?4A@8Q{jiU*{%gkaGL5p>fq)UN?I8tmANUB236*> z`!MlW-gUw+uZT)ZR(FVcA5C*A4zsECh?$+lzGh{Gi<2tJH>yxXdymN{MNs3bC^H6T zhP7CO6dAco8dst{SqK)4d9qNb#g?L|Bp05OsEcMa-hGlY+OhkfLq0!?i1hQGm4{3m z-f3{sf8Tm5?v~}@AJK#d>LOt9V#arb*7;ftiMh2wS5AORo|1WijlNFNjzD9&yYvmP zTr6Ywuxa>8y76biR4AdZ3nqDdgw8#0pE37;{E^mh zvNZ;#1^qP*e7HXk-BLoYM@{m>wA!EnGGg#kf}5uuDZ%QCO2q+}L#8ixiv3MWXxrVp z2l~JKSFOc78+sv}9xG#t_6+qPWz5H!Hmh~>fagAt^RhxvDD!rhcKMrEkMFH=@=Yf# z%wa_uN~lnVxWidIEnxL(C3`$(wPsn^${f;(ZmB6DAHO0xZ#-KBCGPuV;}&IhRsBfo zAP+gPTIHfFeqB$9!baKfdB`Vztfz%ggJ1xkwYUym@DzE-eLBETNaBkj3B1+X_0F&O zt~u-b=fSxx+NMYc^lZ^V^Wqs~N$mqf!( zZJ?X)YdCIN%Z;Cwl5784Yb-{`>oO(Tg7OA+a`M-LJZ8FSn;@dsG(b66m;5O9oY!B4 z7A3vH!l>gtM29IhzMtTgU#j@}I2)yf@f2QXo$Ot3rFbj+zWJdsBF&(qtR59~`+(Q? zm;&U7r71MyH2g-mj=twwU#qF$G?$<;<9Q4sJmt+ElFE4FlSa}k#=5(q1@TzvYd5|#1Uk7##< zMF*MJ0!kj(sI42LoB6_q=R#twV;&f~&1lN6pNSTYS55c6_*p{_1|np%O71ipZX1OP z3I;bovop%CTg9z0B$dl|!9B|Z z_=`Uf4bgr!eut;oBGp(`32)$EMxZDLW@WMMT43em+-nau?rO8e!;hbFg$MkaQg!FR zvbP^IXWf%zA*f+0AMj-iCMgA@mxEja9vmyJkcFI6-LF4+9$tzH{{3pc@+4g6*78@l z!BP$>#$(KCB!aj@%7G56jwX1Wf4S6S_Ch_haBW#lJ0rn(PZG@^Xp6%(pvsJ&7>)=E3OW34&xZEob1We;xkB>$!Nhy4(5HEr z53?|e@-ob&YoH<-H!c#LyK2w924JB2`5uLWS)HD3Z$u$6&7z_Dr*}KSuK>;%+Ce0V z1!%MHhWvqc65u{BA95l>wI|MHW2F*Jm;^{*mj6X;8CyvHip!ou{|9RH3A$n)iLg9Z zVdQGmJKwfeYG{=1qV;QtPz8!FZq6R6FiUSidUsw*)5pY! zE$R?4-nk%h;3A!rn*4kbYbfQ5#~G4r%)p2m8Mfx@<~sz-#wF!LOX#A9%+|W(H&pXbg4hFYA9?qI zZMd3rp2?Lp6~7&<_V^&LX7zvp6|3w4$?Zx(>5@`0Hn5FVAnDPyxr^s-PIO)a!JbHX z#)KxKqi|Nrx>GKNPvAcxWw)ouej!MnlXB5f15CJ$gqXiG#bmt*+b|sG{5*tz>%IK@ znHG`2CSFAHyCLM8)jbg^vL#l+RgPFdfI>o&<(D72L#3kn_!ptbh;A7Q>}4qDIrQQ0 z_1{HAA@P3*WIf$F#pi$5*KDT{;qk&jw~&`K^%f(@|O)}ngcvp<@{9msK|(#*~KE-B5^O|l&9{knkv+<_;< z0$CyDEc-od=a#X;D8--zY6oQ=tJ@GdYM40UOLyK z1K_0C=62>PatTu zWvhV*>2c=q(6wDx&bbGPSuFaTAn=%B@w)Zkt?+0&{lDK7gD98qy06W|2@4@e z^7~Bg(xbBY{0!?kKxf?0yMJSu*pC~S!PeNE%ll&(7;4{Y|E)qE$Lf2ACyUuTh=-!J zy6VRG$L-)mTGYKXl*aVn^Ufuc{`JJxb4q_ol)8Z<^Q2`Q5rB+4{Y2N7%OLbQ?mGNe z_|nPk?^EzQfz0Zq|HWzp`W@B&^N;U+16i+ELS|5e1imc!NNW2`{FMes*>#jlrlxVX z;ci>spohfx^KeRSb=)Uc<7S+rfGNZ2&|-gzctvLsVGo#RJId-_p8#?rs? z;pBZ)GWsSQ31C|b!s)}|lEtFLMpuUI;+zGQ-S$VM`Xb>LVU^ z=AW-rUPy)1Z2rW0@EdLb3hXw}$={PEZHuMim*|pQapmC_FFER(@r60t^z>L$qKZ-` z=-#Z{(P1+9pv0k*jkl;pW7F;Gx}B_%lJ0)2aM&vIS{-SGS6U4mW;LE7x{XFaS2F_? z2st=?FA-9r4AdBsEZHA3KX@3JW0SSPu`H33*!vRqBdfR}lEC5TNz7H+$0vnTzA}nv zSrw31iu+%A?^>Gbl3puwSF+miFTht5b4ew-@L4E^Wxji zEL`&@T*2?~5J|RM`+!7P7D@0;TML>G9l)fx|F;c^rfw5yqH5j^NlX)f78SXNJSBef zS{s(R$CS+Fe7A4b=mzy8aCb&8PydY~i%zen^NB8Lo6m3vC6o=>^xE;ieQ48Voik-k zaEZWfO?-U0yK0THe`sPK%X9HhJhS!ad8J+^%nSIaPo&=oPmqqc^09`rwgu!KER#p} z+`D77r4T?QVKPXnE`hIrG{bQ(Uao$($BBBOQMT4BDMQMwx8;OE6=z+$lm;JQQm}L>= z$?QZXv;~FhaPh%cs358~wKtB{B*fr0M>%Fl8|xrXn;(*K!e3q)-v@r2TiD?;{O{>_ z*wbs>H|fw-5u%LRvD{m&Tz(^}UfD9<>r^M!m;S|{4<2RiuE2(r<4Wot zYhy`N6`WInH%iA05H!roJ>bSAblZKL@SC=WoZRDPXro+YkOJ{`WcfB6Tv z*tYgXXeWy>QA>YUsW^R0lxM(r05#buEYb(e^d*TeVBWSKvjJNyyejJt~c; z-Z=(Ojanhv7G~O0njQZ|`3)$efZT-x3D5EUhtjvl%@|d5zkK4B1B2l^CRGBS9N!%_8M`Qg7YcDE7>A<^rAbt%JZxff$7Fl661s^JIuQ}!>&EQz)m+lI43BaJEdRo4Tfh!s?pZxe%_Bx%`G zX|1CLmgMcy8YXiB*_TctYCNl|5IT{LL@3&fb76qYQmYXz<#iOI>s%ho{;|}q!t>`} z=WfUK1p}I5ziLY?wIyg)9`x`CH_Qwett4A)o7^EBW{-K5wR^dv4Li$Nb9^c2>weTh z)a8$Mdm%FCEW0~lJwh1kbzKY9ELe@jS7IZR%Dc-lL8m6KTB8Mz1Rd)_cnc_a{Ob$m6F} z7Q!AmWQ+73N@SP%V4bfJhGj4miY^l+fNj8bG%^$!#b9>L6HTwXB^0h2B}Y7M8%OMkw)jE38#yaVt@tZ8h%P8X7%>THjlixeuj*X>~K3B|! zI!?FlFWJCWo(5-}|0Tb`OLJnlTSGveo9&|lr0!R9@MzprY_GP(QMuj6#YqGie8e5oqR#r#m`6*{Nduz_?e^gbMifZqPyt{85 zd~>-NXk=`K>^1RV(7m&PB;F~$4=R28+w)TEuF5T>R>Tc|84))qb(v@xjM}30(y@h` z&&gif@7EpW@to&*akRE0>8xJS`=hk$gQp|MA2ys9fs&XA?i|kQsK>^}`sh$UvH(`u z_Lickj?lxs8mMb?@LNXapgTo!nb1(wx#)2&@8c~oc#x`Ki?`+>v%3YI_ zqSl){3B|i_&?eIjrDcI)EzFGDDZlT}2}I;VD%#{qUI`DcfLr9sgLgH~F>CcOnfhG$ zE1VF<*f;tC)3<+}ZO2!$8}Bxe6hd00hFa+$J!`T;L7ihEE z?eu?*eW0XP40G!#&>z>V?G;ODBueZTtq|;4`n~Bv&?J6S4sSxKQwg~ z5cxQJT6nip(+dra7SDh3tW4#(vr95Wv~rfwP?e!@z4-~tQoQUO5F}0c#`g&Ekr@I5 zDY;pLF%a8u_pcH8P}i|&?&uQUQ_J3sN6{?|{|-giTe3P0Bd{pd$4N}HE!G|C8M+Wm zD3vvo3X?TDzP`SE9m{ZjLv)%QMW?WBPw14>SqPixqryZhfZCbl6jI7d;jjK3DID}6 zD(_dE&t>(_VBrPJN3v%Stj!lZGYlh#8p_h2s)umI%}C~!hIml^%=wgepNww7&|lKr zl50!Zi8083RD<8|5AgHo5>m68B6i>Qf1ir3hG0ojy)pUl>8$V;Q2**OKq?84mit35Z(PZ4lp;BQe4CdO7#jA=It|`>dv0&+XD9QH;f*bH{N=(j$q$ zXDxK9h;HO4m#Uxz>~8{6QIBn*9@2Bw|7aNi_@~dtOHx_rKV)Dm{31-?ruuZ+Ns6hC z02Mf)p5gcv4f7~3u2i%~u2mz9z{yHbawOX>Q#1@H4sPC8wAzp3YT<54qCEY2Ps70D z{Bgy@>Asp_6R=KCZoN-mey9+1=ni^A<@b{f5d-~CkTLoUO)pX~rwU%J6ptbD@BDYi z#g!E0Tp_X*HpS>q+eS7?hv zj+vNtJ2Zr8O|H3K40#uT4XONm`}{!3By@-%vJhnS+|MD0h9C#ITD@TfX=D6H0pYq9 zb-G>s1yVV54I0Blj%wT>`@}jZt8t7G;jTN@hk-{^zrBUYK*@SBiphFO0C&A)%o^IatjjT*=YNMY9{iib^{b5@nB1uw9W z<+Ae?FHibWsgnC&qwNl-QcH4rrcWKTzfB*%1ts;~t>dnyFv=2s>}!f}!VJoh3y%7X zVRXMN)w@cvB{5A=U}m@F=@a+ZxX(`?BbHK$x!}=M$G5xQ>d}cEaFx1zC0t5BB1?T! z4~a!?^M5usd&oAKquRT{9XQsdr9v(yiEj}QOK*D9#p)Aow@&OrZ%57=d{je5QkAz3 zY5OE3WTj8|F?+58^R`HmiOi_)OpiYJg?D+4euDT~X_eo@9JL|geI+2gR8;J5IBtA> zq*Cl{>Fc|wc<^_bq#lVV0B3Xq@h#tG5PdBY;McUHt#>oGnJ%g(Jz5io*9JcACBDug zF#IO^v&KJw7TlbYovq8&)`5owwC4>Lfz^?it@&(tf92=O2k}0P@~c}rBk4JasO7{r z%lXaaON)>3DSGkQ3ivoYIA5(|=*!wT@QCnf(rL5US*wsSp~a&lH@C zPRmd~ilM0qwT;ONk!W0%5#$PbXH?)(B@L?S=GT&a%(1hZdhp7Jj?CwMBD=nA;wxQY zrT5?GLdkz_Q-+ox_@T##uirDz2|6C@BrRnhICl5?ys##&vHZpiMgI(Y)rs^Ogq-Hy zTXnE?%6B|z=HY*DHLLeVM23-@cExoq^N?auADYB^UX720|j_WSXb)?q#FPZmY;zRs{h80w@3TZr7YgHz-6Sg$UtMgzRx z-Cm%;!AtV`HKVdIz{%nI2f?SJ1msI1WpY;X3_=({3lv?2g zQn0f(64CUYPG9hA#=~YMz7={8S~E~<0PbZbHJcLdV&TE=oukHIGjiq@?r zd{CN|ciD5p{PGpsKv*8`O1-P7rKQ!<(k2LR08Xf!zH`@3t+4}@^CwuDJuX}=umoUD ze=5KBFd(ZV$FAvgL8OPa&JFGrL%*NO70+OLzy~C6UF(HbI!rf5{xUKnmzL4m{1PD@ zcf0OX$Na7c>U}lj5c`6Q&%7W8qGg9CIJ%=XVa`iGVL7J3=^Yp)HTd0Mx_1k9@hvI*|8IS-c*-w104Co*BU~cYe+8W{RqfQPB zrAU9}BEEF-2@PtW_M|P;#YQQ>06C|OIR;;GmbC*GZ3VJQ%EA$h20<(n9sG~UTESAj zFV1qGm>Af9%slb2(rYwHT_1(7b912Y%S{;kG~T)#-iG;0Dmj0A+K@>>DgRrxAE{Qn zw4wce{OA;;>WIUyIl@`zY=*!NCwVLQ8<&>_FonU5dmbBb2;dni(+xYv?G!uu4-@n& zwr0G|(3*?@Pgr3Wjv)S~8X+YBJ$hYfe2sPkOj-D} z9yK%z0VgofImQymTwic#V>J24EVz8vDS-IK_hok`XFO_5xJsJLCw8ClG9*R3+g`&z zOLclvU-xKp(v#o*=XPJzPjOUJ19moaAlD1eto)Coemsx zP78?JKc`>I9j@b-^lc`0sbF3_Ud!*7w52o@-u;1z#D{04VvOWJ*QO%qE^@J6SA#8O z(~=nemiBkPi05|l5e;D6NUWoDyx7`R1ofv%uCg?C3G%GbGSSo)o`=j!3``YyCyf+3&qBp}si(_QtkQqL zDzhp;73J*pU5LgxS%#AVLrHB_-uV@Te%Y^jEs|eeb)+rOT@*pFy~MWnFd&AYkVrs< zYh;p=C6_z&9-|R&F{?(X^_9jrLlRqUOELEF#&~6cKMy5wb^RIpWA_^fr(KO~YfHn8 zKEvcP{O|%;ivY~+mik9)F%p6ud2;fmom7bZebI8W;BM#FUtn&L-A1_H<`@&Z04$dE zbG^C>1N;vX&h~to*8=ZGMWcCoiD*!5uxw)%QYyI2@jUl+ide(a5sPsD($@{}V+-QH zqqAZ-I6gl$@wT4&T(cwOeG*I|CdC`A2F#H1kJ}C-g0-r~>>AvU=w_4(Ny^RDi6$Q` zQ~uC<92g0p;)-jgLO_{=_vbV&Q|uaFH+da}v)wz`u#FFLPBMOGQiKIGXOxb1;u$-B zo<1p;#@S&T4GZX_3$#P#8ww?JtC;_AurHR78i%Kp=X(EoI@E@A*T(PiIdQMX>;iAM z|HBD%J9o8o+$6B)OKRR5r{!)39(Q3{$i^=H?ANUSlGN{oKv-}M;_+$d?SbA5Ylu=c z8+X}YCv9`_l4QXCfsx~B5`{cRzj*z4S=e!#?T2acP+sUdW#GYh4QZ#%YmGemNd1J< zqXXVL#J%oE4OTPoiFp8+64;oCx9@Wmn`g5Gf!#O^?(;miYxrT;eN>!Q{J9bVT9=Dt=;2qyKrwgIZ|)zh%z`9qx0NJ&90%`eqC`M_tQU^MMa%?x8)v|&M#X=c z?PH@gbekc3h(Z;5v7)S2x)PC*zlWdslWYXtN9f08HFdb$*4uX(WwNXPF8{l3Tz+Z5 zUw)YjlB5iN^TR%)%_{5T@sZ4rU-yRC(=&O>l*eR~;s}Mc+4D*M!bH_)!^mwKqDMSi zGJFyA+1PW=H#M$Woa3VNcFluQ;}SmC1+_l3G%l~Pu@&>%w<@-c7=_MLJiQjy16T79 zppkvBQ#lQM0~H|KAo(Aj0}}dkTn8DYW1L(uUYp+rXOtVcoX1ZPyFfRS?`Bvia#B*9 zkD`Yg?UQWu34Hjt1xG*ei*ADBhaJusoc;+?E?So?bZTknlYuRB+UL$FDC&SaAM=EQ ziBNR3Y{~e_AIYsSqrxJiZwB#gO?%_t6RA~1IF~s%2L=ti(8K1guPabl{MvPgnmp-g z&b}S}qN=+d3)v^y7@X~j;qesVVT=5j9idz{(JiK!1UP~sP^DFg*RX6(jUxeJH~TeEF^xbxWD{8_9tvDibi3cKrK{)n&0 z;b$?YAy3MjUeaKFlTILP9HZi-FXa74J5>y1&T(*>NecR8$9k9HFU5pgLv5E~8N zY9jm5DOi6&@AJ9eE(uMueD^QKz{nHr6}9&g0n-c`?dZW*pab)p}>o+r9w z#B_`%oXfi!HlfiY5fS4Q`0fL;;$LsTg5Iv5xBMRjG1UAYLs$sgMVv4#I41T}elWDh z#iXc6Dmnb!^~-(1W!36~&>8(#A6)(3h6*GScub0!&izUMaJB$WgOnj^5X@L@`4jFx zEi9wrsI8om-ixX7c^p4#4AHWDXormkOBeB@(`03N-tQg-ACC;1P~zS+a+&d6|EsBL zvdBrvjn)$wBc=gSzVWt0V5D%--n0a+d|fIiWRlMd#uhA7YM+jDNZf$FJxRPJwjN1T z+%`}_e;q(u-WcuQPl6icFo~jlh~7Gl7v*e^sj7b7vTgE5Cqyf$erZ5L_KQp-nxrrv zRh%hN4=>%>Ijwh`55s?r{@nC?5H^Nq?$tP<$com<`BsRzHoh>jI{*Jjxf8!o?8iq4 zh`*LIDTd2TRdxcSmY0sLI5BN6yv6!FY*gvpZRZ9E0OiJEPv}B6`ey3xh9r@Tsb*D* zG+PwhdpY367zuFSXFS|W0w9na58Yj~Cu^yv_HbixyoGaxqw>%9}b46fI zQ<~NKtF%(TrcQ_)KM*2En6j_$%I(X@b|pjYn5GOSN)jIxEJiK`ylI2Czfp>qlDlUW z(?!VnqF2DxMS{FMksP^%6uXR!RyX^lT__Ffn^`t_-lM!Q674qB609Q9HtarIJH5Z> zXvSAghzv?kT}G56iK-8)t`|E+C`rx=`k`-M6 zaMa{WGF`^brq$TS9hG}anO^x3Dd`ocE_av!w(+~K%&|4rNY`AZFVZ;=t4R@2k6?&+ zz)aqj(HRoZB9j>6D8^UEHdt$rz3mNpr9V#zP{FO_4h$U+JbD2AfRpnC(@6~opEP+Y zR?KWBe1^}K?qam>gD};}km$|iCKT-2g#<``^f&?}B%~@ED{^0(c>N*m3KJB-L(3R} z$q5gHhPpTDmQ10cS>sn+tDuD; z6K>}AZ55}|PN#qKJMp_Bcd1L zkO4ovYMl+p`B~@xI=xbuc+$S>io%7J5&Y0E3eW?wW#aZiuA@3pb;DwtrUb>b89lzAUswFmhgT`6l)vh?Z_AXH0g|^BD4AtOf=Qyf2wHN#7~M#O?B7@h z$FV?D}oD{G2c7IM*e&-%9Z6<4kbBwO$~xc!>Wn)iCk0N>3{>^qfcy_Y9xr zsq>EK2(M7RVpI+t0tfR223_2DI7McP`LqQQ;{8-BbqG1!q7*?BM*CEkVGPiEWi01 zEdJn_=QZV*e@;raQ-19XL{szMmrq7wCc0I4zg3iP8k_kR>iuQ#m+Kh3yVcu#h~Bv@r+ZlWVv#3IL4lz&WTUUT=8*s0)<@o=mqHN znf#~1HL&!^ipFDU`wryUi=(uTF+Iht7dr?q!@x!#NBH|gM)R4kj4Qaru+g3ti+1cspQu^`L`De_zdm+(gB9;{jI8vG$;WJzYnYs6k}Tcw)0j#v zim=~I0*}$GV*$U6_MI}BuvdfbljB?c?xWt)D46v>5RcV*x8ep@nU#9f?yUZ}dz;H| z>l6NMT(K_;UBVj>RcjV}D|DG(g$kmchkf)Jfi)HXhxyXJ@V%ImNj&aDKwhY6iZP3D=sE3K@ma_Wb!JS!ONZ%_zLh_19aL zC{3u;C`e4^a3W_wQ4W&ZG5rWXA3<0xfkfY=JfL$s;*i1IdLVyrD#=akFg@%7v8L!q z`*tX$CLhK;Eb-$mI{uv+)R9&V$^ZL@mo_UDrgTMj*5WUiPft@QF2^Doa>QLF=*Gpk z;O&1zH!UZ~E~>*J@Tyk#3byuNwg6)JxQ=KCO?^_y`PKayvpNmU<-v~z`Y=BAUEHRE zNk;46UA8v7zs4UzOalS?QY1`P(u@QnTEpoK_F~-Kkxk_2fl7Y*dG$Tsfk`VRowaT_ z#w>t(p195a|2;cpIbiC^EH35sXcmHh=jbe`tdWm2ei{t9jeWv2#{9`t$vP8^$!7!M zayyQRO@|ZTnzfYX2g17yW***;ta-pIO%;|;Q}KHd&{e)X{bD`lwc_qb%UAm)j06fl z6+7t0uw==-UA3Arh&n;56M9ckB&I!JDJ>BWsC{r-?okvWMY-$Ri{FXR9j+~L?DXKX z8ofGt1Xg{oHdlqlea47%pM{~MvtL63Qx^eEAC@COK@L(v`NvlH${1*Xz*q zij##A(01o8D<3c^qgk6j`^DuX@#ji-kKF(7Srxjzg1R&}jPXKvs+Yl+f@YUD5)fxz z$dSn^y7K@(sS~daL&@>D1J%>&de3O}zf-q#?RSO<6>Go(=R3%2j|pxq3cs~aWU4O3 z2nfmAcAIaWrRSx{^N25bT4>_})`Wlw6(?P%=I)5C=)QFwoJmzOHWA~crS35F3&IrA znDWQ2ZtNqBS~aFhFRmk(Hu+tmjUD^xRs}WrV%K?uzhE6W-P|NCRrfEHvAj_ZY9(Ruv)A7xR9^8Np4Z^gn;}7A2rIDkJ;SEFC`*JO~O%ZUfYVr;&qI{f{mXhRRyrl>SGs#|i#UWvq;{gL`MhPUF zDS)tv-O+@|k;orhUK&K(<$JM(@oYGHdBsT*4>y0TN|~H8c3}zYv}nAAXuWX+I-T{W z$pQN$YEXT~B&lHZmzx@ySu}sq-vAIUMvu+uEHgE%C#JZOVg>PTcx66%E{D^-SGND= z?hxHYfh10nV5fY~?b+7K@1Pa=ReN1hIG~Svc*Qsvea&OZvnI+^*h2yjlD!6PULeeV zN9HxgU;6eRtyC27AF%|Bx_^cUHiHova|~K;BIw6FLI;bgc~TMp30T`?6mMQCnk1aC zh443de+YdDbUQ+pp?CnuXr!OcEC3UmB?H=qaH z_8L@gXqF@O%K zk?@;5BFRxSxCU!ng9mpScXztKe%_k5YUbBeP0im^yX)+;tIqCy?t5Kp zy}3f96INngs6p;dX1`aJIifYdjK8c@)WI134L{zaeik*UD-e!trZRktV}Ypv`b8cS z>MPyPyt5eNepWjaA(0OrAd` ze2zG^3Oa|BRR>iko!2oi!6Wtwva6`~Jy@)pMsj#&n#`GFLedu1mwM1c?EdO8>O$X8 zwE@el1x{mdJNID~Q>(!hjrNF53}MtjPVavy{g4r3a<5yfD$<3;@z8rU>OHd9 z8|11&+?|6*hHq6l0{k2}Y?XwnxVvb$sA3N!!KS*U;lhZ8`6S1=?xj$L?J2S7d=#Oncg$TEh4&p9kYl$TnG$fr=kcJkle~Cp|uNlb~h&sxEz+0 zH{c@OIs|g;-1Qf*l`qjUQ!JX0*r(k7KK#VOt;#v0dH8Uh`@%=P^S%8s<=IRCQ!_y|-HBIbhmid$0dYnmZoQB3Nb7;-1$e8SdcEODEryFfO=8tkWOS$QTj`&}li7ShGM zD;E0%l5hfOOCB;eI4$rz@z39Jml0Wl4#?}cckVuI2`E*6^Y zDSws%9OzeaUWuDqJSnQE7L_`T#~0TO2UI?Eg^J`sX^%!0#djZi$SVkwP1s$b72UlM ze6M&v=JuN>==>tIhuKanU@Mv+4uNFe#wj-HO0(ZmNAPbSnBXg$eagTqmIZ|uGMb3Y zK$w$^m&3NDH^(QhG<*Y5EKH6=({g#L^rR<}ErfTaG-yh)0TO&F;o zN&uczQOY*(k)lP)^l6T#LKuE)l0NV#IPExh?NasKWr6jSWDyddytiAU8DMaRTmwNM z*}Lwqw>yD9J1S=X`yfcY{-tG`h)Zj;c<XoDQofbk zXgO5wm1tQo$DlFsMo=ZPh`B*sw-^0FMbN>GZv1>WpCyj9a&t@kG4YmJxuE6))nV7W z;r_hleW%;~wow9U?t)#r$XhwMHrVL)|3-I!Aa=lo;0q$OB{Duflj<{f6DztQ`yBxb zO};!R=IhxT(rrJrwA|=#v6g`EMV~nYLnXiCKc&wz*#$|{EB2R;HS1;cI2as0%2$+~i4Yl~*kugl*uBoR;svwXv zkR6VSFucr|^1yEBKztPcSyuZhXO{&>-ZzR{uG@AJ?PE6q^Z!hTanxgi)MU^Dp+tcB zieZxesYa0Q_oGriAu2=nKnDFLFO#v?#qOQ=Kjxq~{7V-56!jiibRh+83WhHP2Q{>c zF7TQ>E)ToepNm3)jvu^kQM7~#%}(W^#6)*TqTCP|I1!W$p^=&l$nFyM{vT0(i=;sK zeYh+-1)21nW{|x9jyBn*oSU=CjzW%!uOXs67MbfFcX^%>fMSsXePrRkxh50-?_5V& zdXa|FYL0qnva+0oaj~0sL=PUOR{p+}t};^8S5O(6p?3N-V2hY@I5ax1vS(e@N8K~I zePzlji;fA}nnpSd5jZZ2upO;SiVtRFBjeS4cTKFt4SR5@V*@uC*dhh~-xdk;!Y&?p z#!ZyNZlDe3KaEbFoEKCXs@V}v@#j5kjo{w>w??GNKto4(<=^Ym#Q^qbHNl&!hj|f5 z10aTjaxi(80}&_DQRiV902Y00knWT@ujqETB0DV#QTUjZd<;QZ&WAyIWdk4 zz$kSG8LwceP5NTWH7NdU?r*4J1T}W$zUp8DHLu!!a#38qlqWuEc9Uw~O$mNfV!uS% zi0mjKhCITPTwb5rNLE-gUa>$JQxZ(WU@r3^lz#9pZgD)%cQi*jX4pYI@)NogwTttMBlTt! zm()7*=4RZX>vT zmrut4lQxX&*G{|^vg__rD6jr_yT!-anh`E1rNh2vSiUJ7EdCm=DOsywr-xBZkt&9-*4-E}$nZ8l*#lWlIM=u|3fqvIzfxbL0GGW>;a)sS#54?Wn*R+KNu!z_xP zhWu|^gGAW(T6)oJ7?b^e*gY32+zuU-5p~?&+Akyv#wSbtJD{;l6538I+93vw3J=ta zXkI8%o|r96SwuWp$l`#!leKwoFACwZQ=S58SDO63pLOn5?Na$?#T0rqBVb-^s7X+q zU?O8aA)+&;W%u^L^A?uZ39su?lk?x3=@eT6Y`Y5|KOPZIT1cwAnjo|n-(?TBm}FZ( zu%|T&q!ZDL^6{m>Z*w?6;lB15z!R^W-ls{ZCwTAJ2?N8et&7ahWYOdk^WKbJQ0d?6 z*L1O*9nOYniA#5o^J$N``D0Ow(9@IDCgwU-*biJ~y`WL8U#!IWF6i_L|E$F1JNv|u zk1H#vpWde{$J{8>;s}SX`)%DV2yqzAK3lc*JuTJ=ygsXCbe@VFzrXQ%u|3@v6>bM&_Zms>*Mia)LDop5ks~39@zjW3QW@^-oC^4aUq6SJHJB0aXRk@a`q$M zR}K;speALHM%LEEvxj8YLXANTmtFG)@8eQjQ!&6~8=(C6t5xXUS!>yTUK`ZT#-^e9 zWI>Oirs{mTQEIr~6LE3!U!GqB2c)2(d{yBr;KA+fLFly{R}eFSI~^3y6AW72Qt&+HpUPU|?Xxxdd!`8^hSjo-Hp{ z)@UE+Am#Y`-n>+#w5226T zZFi+O{D^SindD&EsK?Z>7+*OLUA||J^HJE~A3APBf|zNmFnq$f*}{^%V+wcen+D3) zLw*xqT-{#|!_T0!4=nt6!k;f}dWVtaV@EM|M`nkvYz`h}nkh*GWd5Z%x;)3Y2l>cQ zU3S;&NDELYNy?Efl&di&plX>ujNiH8qPfw(aZA$i zL$~g#mjYdjGQ_T;w-_mi3*+z|Do25Y+LwwXU6mi=(pYEc;a>DF#$0TJis1c#H#$pO z$!lw;H`^9@z(k*59P%taa6t0i%!E9y(l@hQ0c}QmMz}Cr^w+BqCs>QHVu++i@jSEn zgo8?*%;8hZ4*~gwHfPuoU3ZhlkRwvyQ{=z>SP5L?H0h4k>^_=Dyd6BZRFM<1tZGu8 z$URGx)DkR4)DO<)_K2bCC;WCjarzo4u5lc`tD&OkNF>Ds(1N;3#I=>fA{^ej=6?tR zcLJ#f8;DJZGlP7AL~)97h3qlO3!Byr@2K0oE8BkLvs=$#rXi}xwz#hg)!`wr<|kfp76KgmFZkp; zHDy3apoX5F;T>rt7rU*tp5Zx0!7cqp{~{(+5&R+F+)UP3_;A4(SzG#e4!n0a9CDU4 z|C-gMMzY}e3d?1*x+0L z`-^0(PE?VaV+y&WI~rwIkggdfN^SJ{`uD#YW~{aZL18YRaeF^Dyi3ABHziWe+Kp7| zBG4`A;X1Yml8HxqkNH_5Ufgy&Z8sq4K|=z%l;Cw+no7p7b!3LK3`piyAI8hL`YzlI zVz+WSf#>N?t*u!^GB7YeZjqMcNV<-}dvdUZVMuw{65VHy|0#a^4*u)tZx%t1^jH@u zNg9vqqbhs$i;WcD@geAwI{D!fVVXy}Rnr8VP%BbeJCTFgi_lx#&rq`XT#3-J$r+}Z zNrmxk8R9d(CQd<@bvcVHKOS3f&dhv}A-_#fJho1zn5lTE3W2Pp4=G-UJf<)H`&S5v zQG4V}cvsRx`&37=h+Vv>FmD9Jts^yRa;sRsnKRNhbERv-*Kd@*>7)^0Z|#XZ-Ic~^ zY3)Ax%zv!_4^O?d8SA#DEzW)7_433Xz{Hl++I{6IY&T1v;*1%Jq({RXB#Zuog_*S% z_{~OErS7^z{B;>}b;wf{p^4yQ`a|OvrTdb(&x_s$Qu%aB&rm7?qO6Oelq6jORP6eZ z;NEAqy1F_fFn{KG+iLaDPj1esJjLqoJnE!L5(b5is zLS2QC?fm?&>Cp0e{_IO#;hjrqd(kGWC9e{VUWBVCUqmKk9DQiYuV2fsxX$vot0{o4 zV5qPSW=eH_*udB6`=x*87d{sq_Wwrk+Qd3^GXMMz5ac1XnDXlNzdG4m-uXWM8(oUb zVJiZM%JjnPVbyN8Zkt1)4iB7+NjXwIu#On~Yrbl6N!cHQnZ+LvRBZ&Jx(^z&lK}bW zFO#3oPau^5$0zYAB;x(lC^L4^_~;|q%JL(9a=c~qX$pZ;8m2=9TZTfEUanFD7Y!<}p0 zpk=a*xWNoLdq0&wh1b>S>JNQk1UOhs1aB`b-!YjaL)K}Mo-gIfmdz4G9gO~*;=de5 zKowr6<1!V^?LRqif9?Q)jL&Bi{8G|<{c)<-kvG3NN{8_uJnL+8Ow3L$vMQ1p+vK-bV~ zdPn8p^RUIjC7owSBLqh)Oh(5S|EKKe?Lo2sQcf6y;<&R)zksd|s{W znjML)AuSN)^9E;CEAdkkh7g?T>R${G_GtS0;o8{qymzFjzwUN}POTV82>Jg<_0>*m z3%wXml= zL4%@VP}?kFA*~pt>)| z#OyI}PlmYKxvRfX@0p_&G7kLVI+Inst( zW8nZ~e%}utKc0RJ`~g;M)ka3*n8*jPOf@|I!{CRMpSi3mF0OWL+*|0i2I+F;R3k|J z6o=fwckEPcHtmgTX{(tPg(?!#|0?pAj=-*GH@fx`-Y=(9Ma2-kRK;NNPPwbg;LdBr z1bK#aUG~Maz+W~+z^h|8ct0GOS%#cb`(6mEidzlmpAxb$?DZ+s)b$#H zTs(i7dH7xG;fSvHxIt5m(?}cltISfR<&WJZ(<=h`tA5jbr`cVdf3D)HtW8U5-ZAI7<6L6y?b$3M=aDa8 zAc=V&evc5EzaE|!_$@u2A+#AU3FRyvRtYrvb9<0Rb`2$%f_2tlo?@ZB)%@UUjAT}? z)PQbJs}3BQPC`+89?vl~&WyfG0%B`Iiv?+3btC2`r=>f{#hyh*kTD<@Mvh-A{?`?Q zh(1%Nsjk;A>o)i{Pv|uue7MlFUlR+lyRDO43hX695ceMnV($b$YLOLOriYI=*71Gm z7hREFkSGdWL?rOj;zi94g$ij*ZNhm)^ z-Vnd-_~61IwPppgnqftwshs~7&QnXAPMu6zX$qk8BTf|NT{!IN)&*-qHCqi zvpkyj+GgP0o6$m+CZ=Mn8fL~7@H+nHz9Ew)(KcN?ao&qcjnyk1Cd%ttTCLxW-xq>? z|Gb!IylCi!+@PXIp}WI8w}z~68lWV)p2}|EuxwxHznt;pkK_f`}+wR7TM1QWP!QA+L9*3b+t7WdwIdLEc{cxn(ob4#hB0So?G@C;?G@&u8Xwk3i*DXQ07aCVt3R~- z);AFBE3jW_A_Xu|H4Nb*A9Vr}rByU|AD1`WE)F4SJ!o`B0f}|HY$_T&djJ;7zAxGk zPExsZ7QGqqIcC*NxSsV~ITgaK9!zJaW;{YfTQ4FG;|CNq7F8Pe0Eu5|;v1YIsU;KI zYYJ1kmw0J)JmvW|gh}8tV?rjg&v56M$aZ~oPu|y0UR)!)!9TBlS~Z#pGNN99@GP(t z-YSF#TJZ=^E>s^iq}vi&>dT!sEY1b6^DhU8KFAN-;gSNEU$*22Ydtf zuvN(51M*u(NmZ}x`^`$G{HGMy54@KZ48^rQ1&Mj$ zBhfSq1b;J0GxARAI{Gn8XUk-zyXL6n`(~r}$L1F{snB1CuB#8^yru$tcaO6P0ePYq z@3WpS`xmO(S0;r|xbGnzAOfkFPcZ$JTI&Z-)h~&WvklJKbc4H`L?FlIe*Xn>6g}W2tYJ8Hs^G%%zt+B^_I!0 zvwY)Os?B+-3shH2!_FJTNXa$&3Az);=l&1PIz}byAmZVbgp4u^?k3Ltjh}s3QJS2V zo~7}R>%lUr7(|XD$fkZ{`nm16%BX>kkvh82zBzB>!M0;xkH44ktLQQ3Mz|Yi<;d(HX^>k>?}B6G&WoKxERk9~bj_ZiXdq4#+6ME^I99g0E?*+Yj?7N--@IcW;U7$gEz*!xnX&2Z zzNatqG=$kDmSrY7UP~mD@g@63F5GBxp6-(;f1iD7E!w7!WCZ?3K>v=2@0ZZgn?C6m zj@MCuzVzpHr9e!X{w0KWx6y1b77ge6!youuF(ABt>1ORuLu$Yl?0cd2NfXBiN%jY@ z;h>LVuz#noORvF|A!edc`qZEcxYL$jdF$2tav9RHlmVIgDVRs_69ZR5HXX|%P7NSQd2>P^UPMyFrrPcEKbWzKdO8OQvWL7Fv!3l;4w9D=zWE1-$_s_ zS==8g0f%Cu%M2&k-2-h{=nFLl_@ZWqX=$Aka`0GBHon4m|7%+Fy-U!nUE@-8*z``~ zQq8g+%~kUDi0Llw+mVz=7y&>rK$Cp?eA?R|W<<~?aG+Wf*+!C;)<;|F0N7@C6e*|! z!(+%R2`glh@*CXz$a1;G6B6qO@RrRWCOm*(lx@ozdcci+~AUCo5wot%P{pPNJh)iDI66t?1z=h!V46W)}or2jyz zZ+z2uhj!kW|`8xL9eHSm;DQnyt?eiF|eozlu@{qUHA1B2jUS(t+{+jk9 zi;nH*Ec+dJg0lzED+^Y`jurN?9GNP~vBU2Jnx$zxlHW%Q(q`M)nBB$zB|nny8O9!d z2PkOlHpb%CqM@|Bd?!!DXJ)t(zqS7$=+XjLx;)nH6f}Rg&>}uB9UU0ifCBfGuz9o{ zf}M8S%m9t=aHZ}-;|R&h#Ex#cVYR52GD@x^+<^dWcfDr`&-r!#*T$Dq+s#f*gw*j* zkWRnbkc*&HWyA${i?@QknR_>D{030m? zq_S)spKI1V$JzMup>Q2f&yFjPZ}%aJzp(K;>udB(Vq36)Vct9ajcW8ROjvggR^g)A z7R7s8Cx8N1`_&>K|0PxINpPwt?dOll>KWGg^Px@~>U6!RFIZy(S2<)-uRU>lmDM4X zA6C3sPIj`#vL|pCcxb2>YoB`}{ob8@TiA+R1o+=_zq~S}Etk){1hP9c`@g+}w9?X> zcEJZTd_7V2Ja$4z$n$;c04_rDQ_6>S6&hql$W1Rop^TA49LI`3r7KGHS7>rxfa zd6s%_R zTuCN(p+cePZJe1{UC;RzWoT)~6HkcX<$3Zxfa@-o;{Vb*#mvc5pe^6{#m#K|k5s@P zdtb_X8N(`tRks4bW9EDd!``(U&CSS|RfHgC$FCB`-uRj$$EH!evj;{M=UGCFCh z-#2z#TEcBJ2kY$0sr~8D;6a2a9vG?azEgo#bA*rSq;B4p?)1TLQ%0Aj!b2}6C?j#56MyS0#z5jQ>EKF@@}uN9JDxmQMtDaSWfil=IiVf`$fR(Nyz;n znEQ0+Ml6Pors`~jV1v=TEbnXFPh~k3?bcM?Qw#6|sYM9teZ8>%BVEUSyJLT#ZOFR+ zP3vhK6BVX(CP!g(8AjHk(c9+5SwpSCcU!08-FDkU_gf~R2O=@C3@e_W|-Cd9$ z!>a&?uE|s>r!h13iKRNgp}}|l4gc0A-ty(EJ%vvV9$nxG+)KMAhH)aW_P9Q}QF#HE zQx6;(W~RC2?(vCZP~1;Fd+6Kns$;;F*=ljic6DIeJ#n}ix)1jueg6YVM7wl{*d-0kqx0n6aW!e0uYgu(5;=#~ zJpM&oT~?R+?M1x+$0$&BEpPQiSzx9Hc`D_G@ALUtc~=l?ms6(=(cyhei(+)3O=Ik}v@Ar>pxh&_M@=DI@JcY8&6L}vGxi&yDVb%AzE#s%7lj%Hu18|Lz-5uArx1!8i_rkJ0Dai;PMk^%g-m{(DX|}{ zR5>uWcea&>zC)TRst8!V{LN?`IHqBs!it$kJ!=Kf2di`;h~Y#QsH4As_x{n06pjh0 ztp4jR#Qw8v`EMpMu!rmupQ(z^&hj52wb7Rm7VPStWig#2Ss{1-d1){7H05CG@AuC> zfA23XcO~B2lnk^)#^Bpj3A{xG3nX&5Eg`tIyS(bWTp-_CRyB52==6OA^O-j6AXJbu zPRqSqgq$pSqp&gEDqhR1K+fsZ4kh^po*jL|%KI>9bfC&EwCbK=@SXxki(p&H10{pr z=&i5Y@y6-VPVT>T&s?5l0K_F4qLj_@9Ce>>Cl zQ{C|8;e-=KSAlL@{LTx>kh8MchmFYX!%Pv*cuBV#$!J`C!m6jtV0?YR*Esyo*D(*0 zU|G{wq+J33_aa50Eg5EZfViY!F&umFyZBwyZ9zy=30;Aq;4OO!8X3df>!BhkQUJ`C zqW z{!QukXimahuQ<$@Vze&UhzF=h@;*OHdr$y)Rue+E#X(KQe+N5LHKl5%VH ze}t@Kr%$wX?UQTyE<q9V~?E)rlq+9

    2ma@n z_m(8`qSulAr!^P;0oWJfS)Om0%*MOdF`SKlM77tDN@p@b5-wzBQ zzrV~VUxam~nA}%a)i#9y0x8L@>h&S>wx`1nkZkFWx7XYbnlGe9%?zvY{P(X&{8WBt z&E_W3}mb3uDV5qnH!IYb7)Ao_T8XCmjk>b>8q*n<-k$^@4Z_ zgR`eXfJEke&YIihYQ$(i{#AWxQVC#l*^?8--?`d7*%WPr3>#+X6?4E<7R(MgKkxyt~l;O;tZYH5{chR0%On#`y@P`$C=1V$GLhc4CO=~hb8C1 z!Q|Vgp***r!_U;_<~|gRF~+hIWN_2qEum#OC0mmB%0-_EXBfR3h}qK*6CTaW=}@u6 z7SwRrwUGPr@e^|=A6id;d@sFC!E^?aaz%PDCoS+>0KEASPwpsQ@w9~H@D$pxGOT8~ zEDt%2yVcUG%0UbWbhL_eT8pGjM|!rADi$hkN5%K4lt?u2e79E?p-j(_pKMS16;b;K z2bSzpy*hpjk>%vCuT`Bc(`S^$O;BOq-R1u*n5pgiljzH!RR7tW{MRigEXC}m`J|^| zc~hYp9AGBfmOoGn#6~nE2UDZ+VB0#|na%w7Jcovny^lb0hW0y@4Or=4*ZZF-%a}4C z&-``$`zy{7LxAfm!>&K_tkLWHQYW(-LJYUE{9O(Tjp6j(IT^xo3t^k)e<(Q6XI(1f2 zIgbku%&)g<$_=i{;-s(QsKAxCrT#WmX?vKbyp|LJ`2J#}gpRw_??uS|uk~hdwPa9_ zIt8^Lv{0d-pkb`rY5U227AW@HjLnc6x+D(|$A6_4*nu$olbBWClv?}_YxysUfK%Xl zoEc#V%9ZE3g*BS*{TinIO4G1COf8IFd)PW9^ zv&wGfZ+buZ(x+8iQcv2{jvxFJ;I}1ikIgsxtLwvA<%vU!UiS?pdF)5O7qIt+AbU4+ z?>qa9M2BP?!2y`9v@7#4oVC*>^XQt_&Dri$w%HABli(HXnDgM+y`b5vUF{Iv%b43D zIoWX-bB{_9CP=9JZnI+#g);AG7Mo9D@{t0G`C)YiNM*8U@TNhMRITZ~0#6Ee7J2jO zt#aLUO4$zXIN)u5v*+$Fs_E&;`vQW5Q#Hg21M-~_oQ|$Qef`u|Z!?ea zf?KoUhqFTS5>JM$$AL&g{~>wH9ukaG@3;EA zmA-?rq_rI%vB)ro)OWlxZ*w7gN~%0qUo+KPeu;tB5#Dn@$6(Ykbp8wxoxVO|5$1+v z15pa8cj{sa3#`f4_M<+ji9Mlr>;`kY{f)V01YAANlO7n@sg;S@Q9o~?o`yU6&S*-6 zlwm&V$QmzqNxdD-5}+sr?41}HfkU|H0mF1+6m--6R7~&t6vRw-9rOdF+O|l#c+tA< z2vFsUN5`gxG*rTn4L|qQ6MS$Nt3k2iJY8V^A$Bg zV)rqJFv$Hh5o1aCaB572rnfP3UYjSDWL6oa$0|P>vGpeSCN%y1zX0G1C_UPD@06aH z^C$0IgzqqP_l@YQIyT3Cr83oN8#Ag*m;+Rxoqm z#8oRV^n39~+z*&z6qFLsuVzU9jWDbY`i68dwt>I>Az~!PH4{=!QDuYzWvcV@ph9QBu;0)5v_Xu=;?=Wq#^s zOW%Fx(;f>l>4xj(RZ6|y$)du@U@%ksI7y(J7B1N(F17^wr4d=tngCYC1KJ- zzgFnVAK(9b*t?9s2CMt7!((;I|LP+;{in_?I|^PwB4GG}LXXScUF(Xyhp)0O$6BOE z#65yzh|a1?LJ{*GvulW=;JtRok)Bc7f>6S9Cx=J3AI7~g(N5b)mf?d?$Ia?y(>q(I zaV*sLD#m8#Cn(3LQ@*?+?OuJWfS ziOsZ%ic2ps$l~Gkuxz2s(WnEaj|YCurH}hF6|g-HtF}NF#%NU9A5cA(()}KSw!#;A z=N9BM-mX0Y>ezQQvxiD$U-q?X>smoi*wc2-Fr{{S-?J^us`wY~{p(3ZhC30~!F6gb zw-U6;;BqaST^kpGWXgY5ARR81x3+OnKddcisN_X?GTd-TH#)6FDt3zUY zOZ2sQwGv8hIJM4Cw} z50zFw%zn973X89@hFzDDJwgib7C( zWp&$v7m9J)md~j>JyiC}tS$d#OO?z866qe|StKOGL@X^*_Q!YrLq?P$3Nit2e&JXv z#3X1Z`%74CtJ5p+&i%HwXjkjd$hQ24)?pjG=C9&cBq;hD1DG9_$9BB4_Jys7cq*@K zY984$X6UX&a{c&ukn;{43(g&y9ND-Q$EzJ98BapfMFOFK7xWxqi2*)*u}BQa=J5j~ z5B#Zle=L3ze>}*Il|cG3wwZBx5#~_Lf+645yL;C&ymgIeK>a~5r;=)i;gyb>|L>I;hyv*M0d5N$K4x=qoTGAlu`fl&j5ypOXJ| z=bV}<<3Kk^mvXojrYt&kBj|osE80~cah(Y2N&Ha)%Yy1Uia7}#7DbmtKbDUBS(IRd z-d_QuC*@R8GWYWQAdY{!6g+fdvmD#=KP?*m5ahv1M_t?}(bI<3EwiTe1s4EB-xtXy z%oowVoPla%WpdZlF0JsW3U^kfS@TQu!L0O9u&5Y|s7Wzz^Fwwgarpb|Yy_X4 z%2>EAgG{E2bxMbfsHyG)Dg1Wx)VV!A@W`)7`dBVx>$qIYF^vhV9ZO@Q%u7jj@JW&M zW@&Fce?(&77<)b}r~QDt74eVU@r3mP3Pu#Xj!dn5*KeEc#hOXoU;}emeaeOBqLOv0 z(g@2(lJ(M1K^UFQ@!0%W3wq}LW^2D<1wfPXY{)0!kslke#btE6nAfte#5!D6 zm=xJF*9r5SH$)k`gYRCH&Z>CNV=Wd=bE1hxtSL!Q3_d&C8?t4OAezOniZefg078pf z9Lv8CK${Ar;N^8l^bO((u&b2>6?i{?>b~?nnkk_}QuyfX=l%T%x+Tr^S}^nIl*>kJ z;M9%PS=RWwN0pw;k1fi#2Grd6ij7b_R%bF>X2~!3Gypb+YYDxjc+8I-o{-U|7)K_~ z4skP>5u@@5ra}ABe!nk#M}iAPPoU57zeaB$$)qpVi7(L;+Cc6QaFolM(;bgQMUm2u zY&yiuB45j&A$E20zm|KEM(g|ely(T2e*?XbLu&4VrO+Q@!?ptxSSnY?M;sS1>cdU= zPC)yxoF>dz5*!L9 zQQf}sQsZWqDJW`l=#q2ViUvdbQ8D*P*38auRFfb&tBX1`>a$I;)C}1>{sC=uv7VF_ z_#s#U0~*hDj}vn_135GE(vWf!2ySG}+paaG$Olwn3B$@NcW z&TjRQ-3F4JV$}31-Ww1ckY(Yq_8dYD^MT?gxG^Tl!hBdiSqRJtv2zlriHOVl0Kj@Q z!iZ2r*BY6)HmXI&rRgLfNm}+=c&^;l5{P^}TOX zK$(oNBQCjssGUvX&6C9c_sg{*KW@aeCZRjk{2Wp`WLX_W`KnSY&Ajl*frY-|FvZU) zh<7*_|bkk?^2bjy|oI#uqPD>OM{$;rlNgx85h%d36<(9S_k7m% z9%?#>+uyvF-h1615f?+ZXe*mtd3j77B2qCQVA-u1L1s#=quDx*djkZ@iNnYnzosr_wV_PFI z!0Xw|Dn(+6_qA8jWl0jXoO5B)MQC6bw&DrzHw3W-?K)%;fs!iF@Qf%cpNX4MEuR-)>0CG<5}`Mg1ML3RD`X@qlj>j zwI0O|C<@JgR{>FReDG~z4RS{L3H5W?^am$}Yh7lW{L0kB!Vn52K=E2lHsxFtDrb7_`>wm~;%x{s90 zyG4O+)?7z~C9^-<#E;iNYn~x|ZL+P;)P6b`m|Kpg%45ox7e@(SSJ}JGbVo0^j-#xZ zDc=Wt_5N}?+>SW6%lPDf?Un&DC&dymEtqS|RS(wi{zSwlsdBA19H4rWSM2NNf_Dvt z9X%kJArQADOo+_gee|CE2fe1MDM8K-IRj}k2=SlJe$ZTo%@#kd9-h;^F2XxMsxA|B z!j#jPBK}y7>dvNX)~vR~kv&QXWlGvPPhv3q&2VwV0OY?bIfhnS{Qr+D|1V?wFRpLI zHmUt@jI0FG^LMT0nR}!0Y&GUY?yG=#T;=JmrNb(Ex<}Nmfy?v-vZytggBCgDDzj|- zpjP+uATVaFT(T(nct@>R448qTK*MC%D4jPU=92kuQ^wM(bWvzuPu2JDkOmUq6JW=TT zgz`YW=_@W-+Dk;6Kq-o8GK{qL7eVD{vn+zlwqydUvb9d}>%mh5d)^6GO+Y#Rl&#%( z&m=HomSMhy7HlPN%4!V&}JBp!o4s5ev1tLu|Oi+Gf&_OXuWymkO33PHAO6_r0GFf zF}-{vUj%wm!c#G>UPv7HGr9JN(HK%I13$mO^RyxW?OkY7tmjIbu};?sg7GQ3V2~rWn4%+wa`0*BWFi(o$%N8wPie>1-Pa;*j*<`HY5fhi{TyjjgSFEYb&2q7%GzRel8KCY-f?yOJNUyn|VA*S@8yT z-N{NgX_mk>LAEyXX4>j=Q$ZbsTNq||2xOg9DEEW0nfm{Sx3`LF>kGSnakt_wrATqN z;;zLD#hu{p65PF5p)Ky-qQ#-OySuwfAS5UM_x-*x&egd*x7j;cYwwY<*V^-$&;0ps zY`@Bm_hz}QRM1Fa@hhd?(VA{r$$YFAi|7d_5i~k*bxsPyFV#eEB0f5E%zr%V?+e`C z9DR%yj&jaxFJQezrWH?c5qCsgR|36bA!LXmy21*i2BM8#%Ql?G1i}~N!jLOrP%0&$ z(YdRrdqIEIJ#+O+yQY}YYm`TR2Ya9T`>s#RG>o&y%?d&zde zjYr>193^XtAIW{P`16jO0U8g3h5RcL%%{NMTh{~1p>yqVmjL1d?FyX5COd?e?ur+fos8S4dX;S;VlpMw4F8%+2Xd0!g%O`1# z-?DX%o#0)}`BxuntjsBqUHGsG)^S6NM1dWC8HS8F;(;lJf>CVOhxKJg_cMNpxW`!n}R%z#J7$LVzv;^X0y7ae?_GD zu&>?GdDNC=@~sOI8G2BB03?%sta2Z0$WG=Cx?=d7P~o{kqNHDm;MNJt`z^K z`;$eb#5S$hD0#!OsUHkfm*;c@jKR2pj_DZnKK=B%%0t76K`l5Jj&p~Eun4&0ncSRd z<^njU>+pWf?m>5GNY`&@@C04VF8@FwNpJ_K1@}H?`u%#;JSYjr{qDhOMq=P#QwmpF z=ESZ7p(ZC4cRhM72`40obXh+%43j@r{0;1IGT-zGiQ%qf&Dclz6M*@kTEEsnN+pv` z_aa+e;?)SILras*<_#K|XA_fi{~cr~Xn#C29QxLezLBsPEyRPi%U|3b_Sl`Xh=*ms z0)Bg{)d=QRObK;bQtIO5>EM-l)_^7v%*w~wc8!WQdepSM&!H8HSKw-2$HNFk%?_7( z{|m>;#T`M!C_8!DlLN0l2ggL&?;{nHcIiy{Z)+7LGQ1M$d{ulRP#5Q1iLB%y6N>=D zYV?bbvekSr@vh=39Gvq-sNOf5&oqabWPPv$*$7`*ooiB+K=5kt#^{i}hTrsq5+B`E zj+4Lf9y*~Z+gKF*iO?djYgP!Vz*4yuiK79ps@$*&)s7fJfpNTI!&LG#>17j zgwC1Z9?^2Tt97G$GdK>1`OEasOt-zEn^A~HYWw;|0G?xD9 z|HTebVM~oDU)0>TQa?}lY-R)m;D+2x4e1O(G^ySm#CL@2t!ymmpNPe7C}@Bn9u*X1Od@JN1Bmkc@6Kf+PyP%L{n5ID+|W zHhyVQXpy5FzYCxG+>6;|G2QCZwiDl(`Isw_v)#=B@z4#y>W0wWx{4Kgqno4cXQ2|1V*}LoRYz z>M6o=UGnK>D;#jY!y>Ob?AuK9E`KBB_*4Ud>;88=_3vi!2E9?AE@ID8aizPOO-rOU zNlg4G6kJ|dYukJ3FR_=5F#MTOWIAz2_teE}KoX)BF_bg6w3JUfHDY0J*1~=yOQb%i zof51cGeFHCuP87rFFKScQam@=;TiR#Llwu4z4VS6V;lPP3=V)LKn zT#U$#C6*!`x~jfabp)X01b^W&7_1DFR(-8S1D1sUSQ+)|LT5k zbc1>5oN-FJfPZcmeO_^<@yE2o*L$EcdAVQ1sdSN;?pQwP^OxO)`ABCUIhl9f4+Fn0 z$Z54wD~)lN6b};!&*t9+i0wCD0}l`99428i+~?$C|0lzE;PUzs9nUz6DZvpOQ6%W# zW>k`nq5RnFN|G1#lFl&}0i5k#UF$&|gQ`KkUebsgxcGLoIH1~TbpNs^)S#6qy3H~P z_yfK(r7=g`)E9@*SREzRYWn6Hs4Q7=q|j?o#V?HTCK=mqUL1;LUcD;|(xR~+F#ybx z%H`x(b^VO;b-{=qZ0(X@@$E4Vu7no7g(b^~G1L=n&;mD})13zYyHVz4O1rtgL`XP7 zIH5mXGL#Pv3b@sWw+rY!=QbPdw8;{4Jt}o7%ZHfn(c<0t@`aODN(@6h;&V2{P>3r37&_JBU&D^9yLf4FOWgR$ zq~D1R+cj5ti`r+$rYFN+v;^N7El1ok^GDFlf&J57HSg!FO)(LNw=XhtJBy#L7Wd?$ zDLv~&#hi;XTl^>NDXA(x+#OA1jje4U)g+R+BRM*CJpb$Gv4!BnGC(WWV;mfzx2qKX z7dOZ6a7W?UbE;ApC=95H3O@Q=sQui(5|VcB34R?-=|wF5d~qs-qN7C}UNA>TFJJyR zf(Dpq6uTiZZ6Pg|8GTX&T9yM6T z*o`k!ZSw-CAGMOCzrMNhm4?mE`2*%Xd$46`LQEc4d|doqOUep5e^WmiyE9NolaS*% z!y1Koa4$ZXp!5h`W02lji@it)VGJ6%s3C2r>9U0NEeVoo51fPe zwli=_B6ZFq5Ytp;(OnSx8IUMlU*NGS$Wn|Dxhx`;qMiwO&|v5SR~3uyk{GGlk3^A^ zSg{wm^7S!icmd&rSwpIC*OwK(VT6U!EyLCNIBL{dK5B}OE!p;L)GQ?{l7;duhbqk< zg;GH7k2}Nbb$UI!W4HPlUcSi$6SG8Atno+dw(a!#VlaDQw`am^-nMD9tSl6kcCUv~ z^WT9BRUkxV;pv<0uu*xBdqKTVkqAp3>T}zxh<-g@PC3JEm z@)ML~;d#y)Rlp#_oJ~vc*XOhT#HjR#=mqWv!N{e*<>~c=5C`r`O`g=SG_}cWRzBtJ z=DOb~e=JAUYXWRcwZu_E3gx*&X}Tff5uE*B5_tQHBTDXprIfS%yB+h9d8!9hlDK>6 zo_90A;D35R($NUIT^ReT@Zrg`BjZ5ZRWX2+FscK5xV0bkNM)W?(E7teo`MYB3E4Rf z)bX4pc)$Vh*_V>^KuZYky59#hDOn6H>0pIo^8m;#9bT&2rkPJ(VpJzQ~{glaQQrGJau=S=iTJ@)hns>i^rrYaP zdonpv4cse$2KX4cjv|GaSa;cM$WI2JOBhWV;!EjKgk6XiO=!=h-5jx(_q>a`a*LN~ z;-jqqS{ll$v>C}xl}R1ysAchoTD*HiB6 z8ll>Fk?y%p)2Y#syDVhSP?@x(OG9)@Sc5rl`f(*RDc(PMo*^I>+R6}EWWWqK7Yphp z5r+zPMoj?|L|H0K%g@8oVtTvH8KL3Pqr6ky<)CRB`739iVQEU_3X`@pEDGMw)M0S3 zZ;OZX-}W~zjjdbaxN4KV6N7ez0qzMQ8WxF^!1I}Dvy){1zF*jUwq0)!)V9-?+y~O} z%hpZ5`xsPLF@L9G7Dk%DTZVI=^>BLW_+sZbk=y0+9j%tnYW&JogI}8%NBcsQNj(P? zek<3L+%ogQz(|@VztDWGp7D&V*Ab-4yjWE2fdRIl(=lK_fGx?s^O>-gkfY>zXyk+6 zO`^|!Ep?})`kDDSk%CM9PAX!kyAx&ZNxY$pyrhN}bq;k$AE@{BMLE7S(ep>p?SbHI z!`;SjxQFU5oqrz#c+o~iv&J$_5Tiy3rhP{zIT)Jkmr|mmH<%`J1!jVwySeB>qw`b> zzYZ&#F;5O$0po1pi;8mVk--Nt9yZJUPas41C4MI6zP0lKZtP{kTQ5=0w6cY#E zn$LmQKPk^$-$D;fcX@-GO<(7b7#5`rH1-oHD|)Z}epTRAm^CEgmlk^Vi52YYhV#X# zhMVcDHvD`ab8=oz{)L z9MbTaWg^}qniU`m`(+HW7l9>6x?_K!HvMd%)MYBK`-cNWL^=U{@L3AikBAthqDr0W zgLo{2FxB_A`x>p&En?SKB=@j@rXm>BW5NuE-z6r?E)(A0vt=FZem_vgh5*}gU!^Ad zghA2BBKe^V_ap@!8Yju7Rc+tnks0>Q>6WlgGp5LT_JZvBg~{> zX&c`CHGr~gP?ppPc~^W=p(%|$M<(R&F-t~LUa*u~+iu|8xnuAVWlgevpH8^S zqoHi+ZI0&HLn6hB6jz~%QN-*kE?&WbwR(8t&*H6eoixL*oY$=As zq?>+n2!jZ`7HJ_!aF6J%s=H% ziFnFNk~dZ5l%>Q^N#wUzccp|1@JkPo7Zu!KYO*gGyzCxem);73ik&A`&rult^0!Ez zuhmp>*{ujG`UnN@W41L}l63mFQVJYK@5QL~`#(hLp#s4tOnCBe3e%OoIIC3H6!YO4 z@U7w4-ze9qJ$g&@#yUWluEnL<;FIb8;uVvC!aU*;g*;g2biwytj+2c)xj%r170YKe zX>0)bYv$=Z^R4~8WC{4oF~uVJMAzyAyam?+E?Y>LU(ba>+s$9^R#=kMZ>*s&zjfMa zgpdp0AhET{= z`tSIn(7Ybo5`vnHJDwWeDENkUP?8OyhTixO! z+6PX2uh~Vv;SOEKwzK4t1Qvoiu z|G#U7zq&stvroVF-xi%??tSPfww)z6>HTAUu@nf)qW`gd!3hOl^Pad0x}bbTD}XVQzmzf$zp2e*`8 zrV|R_ibNV?qNG^&d)^*r=y@8QX2ZkMn0|dxORql1|BZ(r zPCif~ud1{@EgNh~)kk7q-Y9x5nchhrq#@rGO1JIc+<|=GMLg8vT63KW)G{ZQRq43p z4(2Kjd=-0hIfLl&sx6vIhTmju{e}EzL*Wgywx;cdd+#ov)cU6;`d`v!O7FX?&*h@-hGFi8 zC_7pf2WMBMqu_IlEN6k)-#EPRdOL1pds6%RinhY~`u|mMRdg}a3#Hu>fUo}E4{_!> zZp?qZvtB(1{S{gxIL*;HnCFh8`^&z!6;*kIK%pk!Eu4c94*hUIaE11`XZ(=yiz zGS5^3MK(Zg$Q9=y&&Ov+V^IQYieWEIQ1bwtF%EB5hlGeaog(9fD_V*P-)BN2Q+c_|!i=ouf+0*OHHm zf+F?^IY_)rc3|mv5I-Fz3a8;-6wnM)UWQBe8#OsLYZFxo?T7rtM2ET#)x_%BNVrG= zO=U^a-xxWloiF{FXthvXHB=+`ao6V1@C0z|&RM>k`-$99@8fW4Wm5r(LcXfW&g&iL zoH?o`TWz&5+Yl|}#bND~sB1oENB&2R1%(}LlUCdB+xz&skxy4a`3iv5z0p0`9vsQ* zV4~nS2+hn__x8-Ci{7R4&W94$AoOodSbb6Pe1$G%OSw@oIZ>m8MbJMCo{!F(p;6VC~{!gsXu z1}!LINsbk86m(7oc@bSAx@VQf#rNhX$~*S^D;_p`F_XMJcC%9~jb?e8MMuN*I2p~Y zHzjPNWwaGS$z0~9gTdbA3%9YR&W=i`MySD2dtAUS(3fp?m0y&flppODu~tPL+;UX~ z9Lv};U(J7bF^(3`nAlJ1)P0eg{B(o`+O2ZgMrfp<{A<&z;WUJfskNH`NEJ&5fYk{?$l{JlkuB_dtef>^*zx7jI*&EHEmQ(+q z_t7*#Y~O-k$fBIx8nO6{%AB3rnlDQ7tAc1{4N*no&cC#Ecp}#%0bl%t&!=Uq2$Rmz z!{HpnY7K2(n|4<1FT%>-x|cvxSKS$q1hg4Oae<|jOJLR+4PFt2?jmK8{@)BwW3xj& zmIzw_${6dJ&@&v5ZVK{v}sZ#QA*MlYhI9OAI84>Yjoe}zXzbbBy zs7awut!4BQ^(s*H@MZ-%hRFLFnr$zv2Lc9tSY!=?wBGN``0pT>@nkU%5%v^=(Wp}C z`*i-&6TR=1_+CZBEkG(d7*g>w9deJ!)>U6U@UP9)4M%`Dw|;~=8U85{keM-)75kZ&bD)|0TQ53T^I7fkz?w+`oCGeGC z3(QdRfLMnbm5(%tP@%ayc9T2tGXsi>Nu66=s3hjD?;EwH6JrH|vfp(NQ47Y?^kuU) zF)LeGj^oBF%lyg@3tqMWS8ewnZ$(hKw%cxkLCfP=ixkeJNW$1ih(@`u4G0SG$(*Li z`*54r0dfB~3&4?T@tKv5xi{sH2t7e9)J~`g!Fk3o(vs8Th<%!0t0ss4J}jDOx;P(Z z514bbSXRk|tX$fKcF*|Wzuqh9#@^{d!!}vtw zSGf{0fG<#h*9T`3i8~yhLrqlAT^16hV4Jx|oXOG}1ml&m*c(RZF_DDR#yl)3d?%gL z)c8VPCkhLAEpF#A3Y8LS zMNs%b%RyUkOQ&W9=vHHfhcg2UVH-zqbW~hYI58J0AuI?&Qc=j&;71zU8z(<}eXQusoSnQ^<+{ zG{pN>TnrS#s@CFc#9y|pHLE$Pv3PK#d1kH{)wTKSwNgg4q$hoYOKv?YPx$8#27JMP zoNUi|TDP;v29ye8?#-@Re4Z&7Li=UD>KoagV-zg>%=r-a8TTOkuP?pT- z{dXAblwkmtJtk*xt|W!-RKIdLgsIb)rmbS>T|wto(ZFg7`hQ=k(kSP9@GEHPNkVXu z=FRqZ@Qt3-h*ZFiFPwlEE#KtK$K`ZxHuCs3 zsQvYpBB%beBX9gfIXra*Oi>*>YItA6bX(k+Y%y)783CW>2T<#=%$5LQ)e)D5MCV9| zz7&R0H)})fR8qc8QOwd1TAe5#r=}Yj6^#GIgTB;Io)iu?q^7R`cqrRKdlDKY+d12& zBWTMXzunTzBCqy$nTIOcK9_P8N7w`Ek6>=HKcQY`P=sFqEjA(Qffw#|E(hNagM!XR zoCnuCj=Ra<4!kM<*e%z?WA%!|ne!!_7H<5*wu>DMsDPKNDHOf0xi#F6WwC!NKTyc! zX{wScvRWo>yMpZ4`y7Ny1zBB{22|Rp3nZM*%Bxbl%KR-t!HoPYD)6P#U9Jrkmizqp z&lf%+AI4) zE6sZT_I*nUgbslY-w)NPiqa_a>u`9+x#7E6x)~02CmQ@`4{vj;+7sO+8q_0%uBHCB zNnL?0axeWI$E7-gsk=>kN6z;Pc zpPIYw+N>0Im)&RgT|(pOx#Mm432z0E5r5kQs*mhxa($ZCH^5sYHa1D?C}vx#IgjM( z3nsjN5Q#%X0Ie_LY7x+iO!4qC~8uSoD2~+q4+06NZWU7lG>rAC-v^PJPdr6 zbawXWt1X?^W>SHK^!EV0vt$=|(No-4i3AxTipyWF7iukbDrP}b*u3bNz-onILH+>N z52m1bQuEb}dMHc_(dx8m#pt?R`#f+^6yOozYMJcK)qQ+w?tHErk94!DP}?=x`KMYU zuEHv`S#tPOEH%N0cjl@2VOrGUaW2jH4FHNrjz#Y%cvML)WX{aV7u{%=X^5J$ntUkH z#}-na-2!xPoQpwb+!k4bq|8|Uj9t9F4J38u#59cO2b#G8EXdunt>ya{+);)6QaT6- zo}Gf66h>7{!|WXy6rKOBCbwvSoDrq}b^eF>*MG?aey7BH_IRirEsCKb?jcg2b$ zSD~a>m2alsG50By9i^ZzqwYmYl1%hk^@3n%jipM@8I$|@;+e&fR zZ_O#}Y(;lk<{3GHBjF?`@K)x{)#BkI z6HHZ(IE~(SkDEli7gQ!#Hn3mMw<*>vr2H}zK|^XMDtLbg8zKREYJ*~38<_(Ug#Kh? z%}1C_mO|xos6IXBUc;TO6X`K9X0&5?sq+v$QS+4(#r4h*un7p*Hk)3aD~d7>pFwR^uZ5Y zz%|%iX50-`Jj^4#Sc?N5Y*|^P^GIs!Kk5Eul&xd^G=#ea`G^J`XC)c*bJ+3UhRDc| z=D$WM|LJ^6D|mX!A`TGLHvCZe^It+E+y_?|a)Da(HO7q>3FbZ5373UR-Tj>%nD#mi znGP|j7^)V~>stC^A=k7>szOLDmu%}4c&Ee)i*Ltw*cLWyfNovx;i^u1B|l_d@jmM9 zCsAS@5FnM$fx!nd`1#y=0+ai{0x42uKX&#r6jlV1q9}?~!u7cd zt6SQ^AB?xDq=r?2COs!Wdx6BI*`$fEC!s*K{v+fIg$Oh)$ihZrTIpc?OD-5At=lHw@yQh^&``_ zN~r=8Z44g)XrFz%5gJ;e=tBNnqT9D7t zz~IEA@W_WS239LrS;8^w3wpadw9`9UEjsRnf0IwV@d^IiaOdZtkMKt#?lHW{vTK)k zT;QJQZAKcWCZ!oU=6h3_B73`}6*6(QzzP_HrhL$<+eCO zy=!DAPJ=2{;=8NhIF`&RCo@4-X^OkpTr?#-r%#R(!6J9nB4@hvPLekb*v zo;~nm!sEZ>_fR)*_CW1C*@=oq61`CJzM@mjIe#s zZyLro7>@zgpyHc|&l;2P4cN83-0OY=j(R+r!URQL;9`6pU2;Yfk#5M_hT=*CAy{wz z37TU51l)+Tz0HY0DqvE=`#y{z(|^3)SWuH~nO;5x)+5Rr950hydN2wcO7~wAi~7SNGd5{UO1O2DkqXw2rkZ9# zAhB&4{)5ABG(;4F<#Ykk!zAyJ`kLyDyAk*vlyIkxpF%HxzT6fQ9JM2#(acKN-Jiv3 zTPxwXG`_RzvXudR$v`qTG)i}qgrY;6QXK-SF1n#w0(cL_XU9L`py?o+J^QV+m69HM z!Cm~&CY>)Mv>j$YL2&Bjg2FkZqvx^%XQU*xy1W47yP{FxPwWPr|F6%(pznLT z*=N00IQ;PJIZR>v55B09G~a|zNnS~Ao(qc!K;`n4b^%v@)N6tc?wFIc88FjFSeuG8 z5cuN;wdvjZqXvQ?CmYP?>vJC^pe&t;k1sp={yo}qCJ1Jt)x)bB*bTZ(5dLj;gK;g$ zcJyvabJ*kduN~HNUTq!QVr9f_hTvaO?cEWT-ek*ySpL3Xryp|{hI@1GhxCZz^$v=~EPx(?->P@`tG}=+pQ$0&{c4DE~z2a)X!0sW!6r3Fk3rSQw zBuC6QdJ-bSurA2m9^uyx$CAa2ur&8pk*uMM3>zijGs&!}?78xNPHLi>9>WF6#W$^ygwQDb+cYFi=(#M1LDC+tWbb3;iqyY+eq{Dg5Lw z^F;$|uw9iEYeOFTB?*|NXgcblD5*@j3R;Cwjlf>on&vEiFg+)`X`Fg!}vy9ZRAUtr!z4nJjGL2fsbKWCcj$3xeXCFxA8@$Fu=qJQHNLuMf`u)~Jy%J*fGuTx zgn)9OKgyY7yeW@q7R`~~=N94;j-LsMwLOYt2V=OmX6}at{67Bx9{jUCb@>tSaAQT> zRGW4t_@;raWOEL8Deyai#@q97Ro`B!;Y_>o`HH+NVeUHg>Bp`~blp4-G z&L__;874oP>tXhkF3ZN+D||4?B{*x^bZ44I(7NNh6uG7G)RwR8xIaF|b80e%3t zxIS~frg`ReK~=8vwt)c&z&U0*+`IMgbu4vzjcv>yDBnSPPU(POqod=JI8r~;LmXq> zLt;{NSqk9>rwQjV?q+Q#loXKY(a%^dT~KBQ^7J8y;kGF;dLb;-SMA<-wQY(!1a`ry zj>%KPd7S)x;Vx@yM+aL{92HWboUSO_YlruKAfFVK{}~Maxmh^K1VXi+Mo##QU<=pf z^f@mJh}u?+g51?Z7UGgS_y76TRFXt??LQ7=)zMQqE&7r4{8hRGBy4Y+@Q)#NzqYZB zHXSFPmoQGUImluAAzZ8vtLFMG$42xSu!k$g~+$LBr22S-I2VvesI5^A|MXUT`p zzc=c=`4>25l6eg(wukPtpp$skQt*=7_uIiWMY%}EIxEvfajkA@-Qzd$R7X#KZfZnc z8eRDCSdweHl>)U%XF{x|*w{9ifovilp-&M&s+`3jkuNLJ#UCl!0?S0$w*|rS_tBz8 zkK%$f){q_&Au@$kTdMBTe5(}}FOPLO0cD-fzt8Sy-cLWJa~seqxaEvX03$$5Cr~O? zs2QJ?n4LPgpoL=;ty=MpRIvxlHcb`YtFx2+6>0Rmr^k1JAN3DIFDJy)-YfniHkW1r z{BOw~RvELVg;2tfvf}JuCmd3y4CNq@Xq6)7b%~iHHr%LcjjB!d$p~I9ouanAFCLjN zy+XuaAz8!Hg()5X!DTMDY{mZY;bAi2wA)Zl2BIg%7mM@hEQnRi1YU6kpJ+_Nk5qaV zRve|#r`CLa_f7Y^z4`ANUrmdTz`6XDsMz(txing-O=El+3ZkRtqS`2q5?&r|Oy~bw zrncM8mAwt={>F>l4h-if3!idSP{yv(Cp5+kc7D5LA^`h z%7>{4n)q+J-vIwRU7atKY3Ve(?~xz4wY0-AmGgI`tYBsobV%q3RTHGohq(Gka5RhS zFy{}Y_q&E)fXzm_q<0x%j3nZH+u<0O1N=`CKXIHSq<20|I`bOvi&Ab2gdR_|Hc7a`1Z{Qz!NAulG$d$5{(7KV9@&OnaSrcFR6T6RNhjPHV&nYHqL_*H$qhwO<^2l!8r*ej2$%h zKz^(8Nh)Nyq-+vfSN2HxSZ56}p)DgqNg@0{P6Rg3qzs<^;QYrB%@nWJ!)+ucjLKdd zeNzKctzJYXK}*p;ic(5eofJC-8iS*JzCh8Kfzl|{;Ka2Tn#FNF?vE10=Eb|hefaeU z>zW$-%l`XAZFbG*ka%k7xolGO6_dK46uM{cqqC6$zw$h3E{D6((8uCET0?G#G0#LX z^E2!zd(~aI^FKw3wKR*NnY^HtC=HTI2CJ_bqgjh}E5R?fRlfj_Vf5dD&5jVwAMjrT z@6MBJpvT^;qo?z_M8>&6-h_|~M9LQ(Ry%%%EpuZfuXD%Q$XW?anZ5=nh+AFE_oG0z zPB)%C99_7pP-otxldq9;&P1&waJ>sp+reY$>?M@EoM<#dU>vUHT}}PYDgk-n%{j?H zcl->F8~4ktv1D?ot>kYA*AEt53KK$hoUbohR`7b)zF{iYb&cPuC4hN*Q;zBRm~%Ah zTbe=$(u1c=D1#J^DF~A#Rg1J?=Bgz0L;2cFcn6*sF*Jgke88xk8FXZLfA)G5Bur}O zY&ebe&Nn{;s=?2Nsze>Pm6~Zapm*|hGJ79%dk7i_Y4Pg$FP6Lkj8eu$zi$ty2YeO8 zWB#*6)9R@=bPGU&Iq#$C;u=(>MzFrO0MLoQ4cy3;|I%SI75Ka90#lNk@`}m~%4 zPF|AqoZm+|u$cMwN>JqaiV=zQsa55|fGc`t&d1TM>&-Tp*g_^2uS z+0AnOo5t6iaIx#&{PM5=WHhg)(4P-J_zZasC^v34dmV;UCWRAq#yx4q$z8H(8r&XRc*f`DVEz}J)!f^t^+AD$1Ss;N^U~CSS z3rh=}`|ch`_~m3ds`IwDY=FbKA(MGlDcyxgHHANt;Nu=>Th2z7!KC&KlGO2NuE}0sa)Z6^16mrfB{#$PzfuiskfP#K4A37G{Bk(AT*RLV z(P}wu^?pxf8P=ckJnAFVEnp1qB30x%-G{GFT}kdgT#F%*Y59q4(&o}62x?wU9MqwU;@jE)Z2YpOJd+7M+Y$>QEpvs;z_xMgjlV_< zc!xKo=lC`rpa1N=aQX+rLoVKTEOpO~LfOx{>y!Na{8;vnQd7T>j$HU4a64AQC!Yw0JH2gSvd8>AvOtqeX0 zlIKrI#k=1vj>YkxHHtOYO8rvKjTD3wJ^Dq@T5AQH-XYB5w_Hjka+o!D z-(PvEk9UPw$hRvoV?EVFeo_w^x@4=akhcEek^aeEy{9-#UiyPV+^R>E4Ozdr4*?-G zwFk+W+*!8*W0#*vWj~yaK-RI3G5!|MrkGo{Z5cv@!K)UO;c2Thw$lZBH zwSRn@h;pv{EHeox(8>(k@&$)MBOpRlmbApZU{jsFWNK3=k3Xo;vdZaCM`u&?k$h8FkA?O?!X z=Xa7xQl)>=3K!Wwa7r;0@yU9Uc+R5q&`}}#WKoZywmis&7A!CEm@axhYzfdt_q&K3 zDv|mr!t#tBsb(&0Xb_w&&zeDu=O6TooLJVXuOwP+NM+vj2<>-6N@ECLT^~R{DuxT7 zGP>lC_;hBj=m)Y#Q!X!zQ^kuQp#;)G`VUVf6<71_fzr@5C$IuJ16i9P-zn*tLcY|g zOB5|kFGgm7i$SSYvt*tov$ev(nzNf#bB}02PySj~{a%O;;YWjjAb#KM5GQ$XBMfDf zSpKn}jS(~JaM*oWwL-fyZoWp7h=Al$PeH3)GaMN-bo6~Bj~X=_OG@laF-%Bd+Wc?DeHtPM`fmt#S?GoRL^2LO9?9iH=^(f< zZAoJ`lM20|U^l7%OwTzWfx&r6BP<{Mi^q7i(V3wv&%Lmw0#A>!`o)E5Rdqfl~n$<-qb-t zB(<+YO=FRa=T`Ik{T!j)8*pfBrR--l4de4clTf3|IXWiV23cWCV_=_??aPw`8b*ct zVzsWp@An$EV1_%18M8h1uTa8C2ycNzQoFG&xFY`_FZrIARZQTztbCnzyJEbO`AE*c z$mvc`20mph^=nYNEBVsz^NgGX7x^JTBimsQs@L^Fc&N)P7Pl_y!w!E>W_z5*woRun zS++7A(<)8=VmF7+&-9^;B6eKB>FGdVBi^+WL9I|;XRzg~F{4ywc?l;ML)wM6wiCVKK6$TMcm+ij-*9{hC;O+tQLQO+@*6ptRBIBvuCxFvp1z@ z7mN99hhK~qcb6vyV9sP%41Riif#&SZp#(kW8=uWVJA8Ddf!m7(9WxhOUIyB)utn3j zU-@s#PIn%pw3^=olf$@R2CnWC6bf_a$Q%Q3ITvRR+{^utoR_A-_ zDoT_ielIhrT4mm9L_C@rRzjKy^@Qvp&s#b_A`1u_KZaO5ypvT>evJjfIn};7I=J1> ztgaggL*_jW#;&@)<<_=oMPVP89Q=VUn8Y6bLpYhH@#-Tn=fts&eAbU?3pYjK{me=o z@yS-|c?mEvd%NI>mu*0PUG!BZ*Y?dHkiLIP9;T5*nbKEgWwFJG!duW8Vl3HPJ7Vr=){U+tOVd+eI`>Ug-?jMhOO`o>WQO`00| zL!?d8VN1FhGMD17P5wf}J)$^f=J&2&^K##9#Q@jS&!lB| z8y~+N-4$e_^`NQ7hL8X1l$`u~PyQWON@)qFWfXhK#MuC6Smtg)r$Rzi`E*y`@;hHC zCLpT3N4bBSJ^=#`ZEqcGP66?LX!rP^j3NPfPo(neL*)S?+_o(N+qjm;umew>Vf$we z2cN3I7toRWZ~^M**_3E0xzLG%LpD$!Qfkp7VV`8pLSf;c4Tl#+;mN5wJ5bx)yV&W< zf;F~ezo$@>NLx@^v^}{(L&x!a?KAP$%l&d-Y1oH=aa1?U^nFl(H$u&OgcCr&`kma8 z(}|^UyFiKW?NJJ64*Xgx3Sb%l}DMhVv+H)Y1GjnN)G667vp=lG#;QD+CWDx!(?L5?ha8z zxR&yq52ne7h4&e%@b!flud(?=F&!{HtJ5LkM~Zj*ysV*7_sCgs%-fJ~8rJSL2CldR zCox+lLvp?rkR01_Fex8|4)hXe?jN1pgigLyw3}IEWuDV35X(RApBCl+2t<&|4?;S z0c~|(l!pSv-HN-rOMzfTixi5x6)El>+}%Amw79zzcehg9-CdH&_n�=iG-Rci(gN zJ!|dXf;$DKN%%hh)6=ig5d_~}BdAkKNyxovE3*SYcBFRfAr2vwAAfi>v5Sxo`s#wb z>fR1Jke%Nb?MfWtr`R>LKk)3~rh^Pao(-(Li|Lkrn4bV;Y%yQ7kTF9(>q}Mk&XS-U zMyTK~2-$ww|4DW&B}}dKvnIo6wkkOFUAp(Gu3K@5V)TVY2PoVfx{clDIsm?m*Rkzl^2>}=rVJlr_N#$D1gQt|?<&fxV3Ip~Ri{72lZ-EK`fKI+9P?$i z80QhwPO|H$5ieBHl;wIS<+#3askNDUDcvsuyVGC80IbRMN@@5knhFNJyw;833)LFx z!fOl4?iIQSuS-fc4twiBUq%P;y+LM!abTrklP&^siDEjGmM0=vZRZoX@LLBitNT;~ zZp+~kq9_wj#sN^tEnLOu5U{00KUqv3ihWGL5NJ-Ww@|Qt_P8fTdYejcnHK!QF8JzN z1wde5pdJ~YOcvPiQpxk$=CB%;v@^<7E7@N$(;Z`5HeyR<*5s6cJxHJ22H@Pm07$Mt zL2sjgch-0jaOK-;wgc5fxG`drb>lEO7PmgVEU9}ft~1_nU2)UggRo7CxatQIx|;1H zbh~m#e^W8&Z~-pSPK!ikrA$=<|AcGr+^45PdMYx8pLMP-f`)1$;XRKdRHMA92c~Y{#~2 z;Kp>NwHo)7UjunxmnIUiO?h*;G4I`%Gx(HZH&>4RQq=xn5___kBR(k2&2$&+?r@%* zv@yGRa0SW(D!eqxl^DaQuE>7o8h1KXY6YttKnni?bs%7;Cty8zZC7^6=;shdceCP5$v8+Zz%#Vlp#u*bY} zYK}k^$|hX?VO)uz_b*V}2&9$kOGlH^B2r8bxt_f}q}O9Cl0D z)i~k-DFM#{f3+2OtD8nk6i-M} zp(hL`BV^=;o*Ykaun2>pFARTxCUX=EY_u#LArrYh+NF~^0sF)B|MLWW?;#Ldf{tql z{QYrIEvy{#Et6;S5@V}J|KV3DJg>C(0|{&V7C(WZg`!z;%zT9%73(&3m=uPHsthIx zBVUD9x@?}WrhWG=yR~);`--cRPI#>K)Y!gSl`db?@T_l*Xwyr&{y#&pr^E21R0P-| zVMD17@`=un6lM*DTiey*L=niqr@vJS7JTHZ4V&-W#v@37i~Rk*46gE3SO|qED-jDM zwHW%Ug(AcV9%U?Xn+>70-OMSPJNDSO7aX=Hg&AJ9djxcX4>ki%Sw};agr`_+aSt2) z^mq|{G5?}IUkhR*uw?5&un~DD6l3$XHhpE=@6IY(w@I7){qjo72)^zs;FrwXo$uG* zPOIfq?az{|Fh1Y$5WDb^(BTzYXO=I&V{xMgQt40qgnE8YFVIh^0JsG2kB6X=b){VM zp$VO+{xy-6f4%zJIK$tpe_rls4vz=Jc`LNlTLes-M*SoAyGX}28&Y`pb}JFFk9vJR zWI9U6$pA#lTW@6rW#h&P$qdDj7PpRkpb~ptQ@lCUyG-%MO1P54Z8A=|h(b!Kd zqe%A4zAX8`+IP|bcQnn7;Xhg(wC{-(92hLOPTPx|fi=-;{QEFkcCmZj>d+KZ9Sdv%?NV+G~#3@FXbsK%bZOFi&BKtb4%H4(7ul`ms z+cEGo?szxkglR$QI*c|rc*H`V%el~oZB0yf^?4F|&@ROit?c>J0b2sBZmIAD**$V! z?-o+Er6gbIfvsqv|Lbi`6q^*+RG z#xiny@pYcWB=1y0uWj9{)TTpCujWU;a{yR1} z_L?Q-cdE9|l7G67e#FxJ$y9HxY|>2|vWquXRJJdm2l36t*BD!qps#hAJ)1lYC(&%c8FeMR?>{fd0y=+HbRW%IFz`)SGqW7J{b z_l^j#{G1$xX4c=jsLufq=&Sxd3$_)6?kRqxFFrX0>>hh9TOdUcf^kd0o_t)}^oNGZ zhrgUHmUv-cEe<{1FK5}_LtoY?v>EJ3_Yd`!ka>_~sQcZdP~UYBJ*h)X>twNlW?A38 z?t7)`H+qwK@Sh&BG>GzxJcj5IjMrHKVlg!@w|}M>ZcuQ~Si_81CIBdgmPXDBMU`kW zOBg$@LUQx7Q1||>zpOE+h&~oJzRednzj4V*Ca7tov5hCD8~G?U@!Z_p z?3nC8=to&b9!t`V!UN3d8pY!hiuA~t7b`T<)m4NI1)eK?#&M@pGI=aVEC`J#;VzDC zbaAk>W4zvo+RCQInJ{U#7N~_zM*2%kf?sa789<})5NIBI8hdI*jV(*?>sl-?*6J1P z4L&3ea;m-Hm6%OKp3cOsUOJT!UDGZ>z@hU2_AByV%ge6f{ep0D%4Qf{3W^h_f-w!| zF~Ekmyv=eYw7_hjcv##Nk;n3?b5&A4g|HZ678-JL)ItvS%N}qcua&A6g6vNd%E#Xi zxzM>uv1I4{F?pAi{(n|!`>g2#;1>@a81&4=T6CA;DuHo30RvoQ?h7(O>7f`OwNToh zSilvN@G%gE5Y}+|!T9IPchu7jxB3sOBEc8u?CFM{{Htg_##jTLXn}K3P$9F6<3wZ@ zM_GPlR4Vg`H3K;ZKC|j=*l>Yd<93LA@!z7>o4h*#n)hU`HpSRwhf?4alV>@fm=9Qu zNdw1kp}4?iUNu6EG%Vn6m^s*F8BWUthy@u?t_Q;s#KfJIECg|iIsT(4jNqKXdFyG0 z#9-xCLh!<)W;c1>pNpVYH({Fv{P*IILLkh$OQhA$P$pl!CL+DYPOBAVp`1w;EghLi zp5AYsINEG_3Fg^jRGWm6$fqAT!YK&CrqU5&tfA4kiuDrRQ1b5LjY7Z7C>LG|B?ggB zt}poOan#(`HI@Gh>cqe;|Bv69Th!#;fMwJ6HM>Lg$COU7%#0P**OrZ1F-5AV8X`n< z#z}KsI}UN*SGR>1M0JB{e# z)`e9rCsxf=|La@5apc!<8Du_~Eb*dZsKe-}bbIbd-XtQB!O?qVf%5P+5PTk3 z)bbx06{G0(3YH0#Ip%Lkp*l(j?{&$AOvcJvOnMWI`L4N zgm0;#i@tW>nWIlZPGJ7;S~Ar>Ns6j2#$JjEva zxZLRk1&U6NW9UZtCv+*z0mTJ{0USz&vn$LVcE_X}NWv2b?5@36?I$C+A;iaj4F$ik zf)!kSuX4ekkiUeakdq~A<0WH|ydG26C6C=7p~F}5`VN0>-P%~XcFJ`H!rKZB3<7adygF?TNVep{f_3a!yh zLPI)S;2Vd1SDU2w`nm)_?79`Pi5`vHZy(2U6&%;GPeck4Y;yZa1Q}8N8_*U${N-RO zo551wvt%M%DVd$jAvJsTdWi%nlX8*s{9|IjUT%S`Lv=uYF@RU$>1!6ZhO+3Z+Ll*v zj?*d=x4AKg8&909?FZIV;qYZr2)A9@#)dkYE?+Z>bi+5uJtvYb0x>nZV@hZigMj-9 zyF=%7Gq3%eb{_F(efJt8%vAxrLe9-rCD!P8@~%LxWfj2iEC~|BLuWQKnDt(O&U$4&9@H*0SkGET3VzsEZR7F{#$n%J!&|UyR z1S5SHB1URBxz~4_#ou&hVe5$Jpm(P;M$DjG*4+8&)+2>Xn4%802W$4rw|$~U%;7NA zM?OJn*P$<3qaL+d&QMs0q^ff`!GKprt8!UJEjxi0r!+X@E@6e@KV)kEhxxUSJ22~~ zP{*LT{1o$`%dIGh+~2vs&7~#M69n@k2)!72twoly=g@`?5qlERQ~9dKHFIY3(&G)v zS(KT*SawMYm`VK;uoc^~LNY@>hvNz|*kHuKz@R7C;(t&3?BXa6 zW!iBP*d&=AN_Z2C@$IIpT$%=zpfGAMmQCd_j5|)0xZg9Yj`tI3muJ`d=Jhw_=r6sX z{QCp-8F36FZ0}Pc$jY@mc=rRBa3f>opIMS} z%-agxlUa(iOP5-wk?s1n>zYh%WLN{DyA#2?JMzIl*ctM3vi_;5xd=3c(a4-`M0@G5UE(4X@eMsFJd$p z@AoEo+n(TH4`DW;8|RdLU$L&xQpk(;hJHW#ErZdUUiHhZ-S2(f0$;DcT@s?dq9=XG zKIz)6dSVK_)k+on8zA{SeL=)TK&@D=PMhawRrAk?I1P02Eme6`w&4Yxa*Va!KM+_6 z?(J13T*C6Wy|C)k+Y+fsUB&utyl;VSCR=ce7oCPjcy?t@(t*{nKc+L|Cw=s+lvxK7 zbM%V+I#|#=MjA7I_wF1iRq&Yfo3%G~?NKjcQISLLuPvSJUNM-hC{{YMb39&tjZJkq z4lV`h33@JT#YxyIC9MiUg6hrLtB|DIK5+Z@5 zAOn?vg$@w~tLGDfa6~*Fju@u9uj;=KajlHA3vV{cCopW6$MU+JxO$yBTuQ>IDZJdD zprEHEyLSD9lHz3I$XO;IY?wI!_lz;enjhlYng?YCt-iJNG-P+)tY+8F*2i}~uDW1B z-Gi~oe6?&Y!Xb>Cz{g;HzkQ3Dl;cX?arAC7zb4FB?Ai1DX8>_o6av^2Wj{Nf=HL){VrxyX7;-{n|?d~w|$ksW0`hRUL&Wq)W# z5Ji4|E0qAip(FtCf{E!ooqFYA@{UK*wx!-bE6bD6RmRviUk#C?EmAI?^?6g$bGSw+)v@D`#N`W~iGZ z>@2z+x?Tg(NJS)nOlDSP-|Bx7FHsF-X>@J{ixRp_V#~hC=K)=5&+PVo^_o89c&&6A z1r`aQ7DEA9_T6BG^)qLa@j=$>WgKX#pUFhic?lu3Olun1=z%_$6m#AE2H#H-!^w|0 zs#HX#_(!iA*!7X@rM-zYO0c#%A|a{c)z%!)>^j$Bt5)5RB@|=LUe5E#D&5iO`1l}$ zBn#)o*p_iZEc*9N`KrBNvr?{c6NQ1=2g3-ZOJl2pwQig--a4P5$Y7aKDnv5I8o=YH zgKwW2zU)VKPzieTycNj0n*0vJStA}$ZhfB0yBX30rX~s6C&P8P*|N-ENytBGp}BAa zC~XrqL@(1S*t=ldR%p8}N$C4!sK4-~6C3Y|%$MRkYz-UurAsyt=8FGuiC6XEsfj?w@bIbtoJ)N9Ord^ReqW|Z zWExrcEhu?jTzO?8h&uHC~f)N?I zIyRQo4(D83VKnf?3U@`@1@%!&oy&nJf^>eeiz1m0ecRUbR4g-?^yN&y{%SYqT=Jc7 zDdzh(J8pm8%tx8Lk!>>+Umv-&8aIziM%Y93iDlN;`Add=Z*TIm5_9=+xUA#lAiPC= zP&l7qgK>k7OiEiAjyjrbhs4?w&j&5Z<@>lHCiwC-$Gsa9&8#JE(~Y~+ zls!H~BQMHb)iZE%FT3@}Lv&3~DwM$@8BiYP$KWWn|W^XB~v+O70V$eYWQ2 zWn+BLxniU9F*c*Z;e-+7d()QVx`WK^f5utCa=_eIKj|g7YC!!DUU?vuFvU0QG!U~i z76(7Y5jKL4Xso3OYF2!Ib?8QmO%m2Ei}ScoN(bKvqaAA|sq2FBKaeeI&SE0092M?# zDr9~P=IGke|JyUTk#*P=9Cq?8?_e@(v_d6*-b|{qHJ}Ckn&gPh!!{{oE0}4~MU6oQ=uKo~238jbaWCka6lj`7tiZD}a%5 zF%gH%i&~c1F)QAp{tEIY_=SMOAhbQ&aNHnonJS!^4kc{@B2D<2Ne4oE+GGB%SWUO9 zXAc9zJS;CQ@%>9$Sp|t(awg+X28jBG>3>{9K_ow-%{n8BJe-5Jf_T;~yOs^g5_(Pk z-;1;8UH?{G>bK#0qn?NBF%@0mV|jVStSd1b4zTHhf`cg{e=+sR62j=R(mUd*k+=9b z7B=5RV$erA&+ul&ZEGG+nCPv58M%H=9BfhTf{#ui!ahuN`&OUQ_IH9(3%pAlW*^u$ zbx+$rxN#llFEYIg40OgTBL53}VAvvytH9)m#DomXJ|PFW7$;kLVJl|B-tI4mTD z=mmdF!QSaX`=3wcBMx+Xc^v)(eDemO#|DNfrjJk0ABq>5CpTKni4!kuC@oX)GXJWS zdyWTT$0~;OpqM3mb5vMy&j4xwsry&M5vs$mOwg+j_Ge>;L4NW>fZNW>-nxBvEhFZx zcGvH(q+VE7Ab&1+I$Yx-Xq1Pu!*bl1*{?)x9QUn{7({eZ1^cRQEx(-!;mIxgW|$mE z_YxIu*7e%&63P-Te^sd@l^Ht(x8!0&GmX`wR~Z9m=h$=YlSyjqPzfZdf(;0#+qSPD zmX`{>jg?hyKa91*))BCE;3cpKqZ;vHyPC!Ppq3=vpLPy}Kbze9SCxxZhk~(BZG2U<8$mq!ApOJ@Eo{B4mUnRvds0ftv zF`@p;2W$1xPoyrd=g0f1fX~J3DC;l?rFix*Th_u(lOB5$b9}BsO6xrKR{@`vx}F5I zT3HGTxNTQdA7{t}+U(b%?e%>zJ-TJ*WzMac$0Z!lEu2hvLiA+|v_5nn(Lbb2));P8 zw1-a`DE!WAm7?$2or*gkc<4O*$A+E>qbHU6A}plO?)rEhRz>X1mJK4clomW)Rt!n@ z$QfoZDKbC4S!8bERk$k%CowMn!Uc(b9egR%hTxwXje+cpAnCR#o+i@B_+7I!ZK+{F z6F97~Fh8L9zVEhVmyE3$?0tA&Ok}VGgt06E(xxzwuHfGJQw;HIx^S%C%(Nq%kJ2hh zvm25OQ7fuzPV={9+01}&5kGW6mE2r zjUCW}Ee&o^%BD>|iY4DJDX1y|eV6g#1t;&3cg?ELAR?;SH^PHm3M9%xKX&KWf#Se0 z{0fdkT4qZ9f8ry5lmmIm!1t#lUg!?VpTI( zXSZ?oOoKuJ?-UXA#CW|1@W`~TCd#OLjLd4K=gVKhwUt?lxvBu15NU}G)O?+rXZ7Lj zhb*Rx35T2u|A!)%e2F?kS>p07FB*9o>D|i)R{s_Ix$?VKac%*3o2Y`~6U6Y?9cIqP-17$ibA@ zrEE2g{kaekzk*BqQ4KqP3J5bw`+vY)wICw zk*$_rLPKnu(nX~U^ml;~p{oy;b;?2!klUWD8eI(BPzH^7)cDJ|tubESR!8a1oF;tk z^gQ!$2RK-pp=*TRrI5TZIU@_1-7a_%I+ogk;pXI69IF&_-CvU5UrfPF2Nd_&rtf9f z`P=9FMs;92-*?Q7j8?AdR3u3lAz+FJeZAVB)E&0nDgpm~n)QVl-w^%GxE)ZkqU)8- zd@u~&IKTK-d?;4XOcTq1Z-6B(S!kzk@1vLZ%Nye?G?jtOX}EpVuZW{DqosaQ33Z}` zb~T|AK5AVgPWEx|V2>Yq$tkDKHR|{*jiUw^Q&0ql*X@ifS7nZl_5OHJcRuWNkVPSl zVeguVa+tj*QM0cOCv|E50tI&Au&F>iZesLHFn;%PG}8_VK@ur7DTrg%Mx|yKgE)gzK?i@Px2DK z00WjPP!gcBm+jcs6KmDnM?^r!z&fHCh=N>b&Tl_ux~*-NvatTlYilENd(IU%gS5q0ZVCigLR?HW7yo72tJ+J>AnB2ES@I23!RL zWI2-aZgWw|$ECF6BusB@S4HGJjKaOVF%=Hq<@+6k;hA(ZEn|5eI~`)p&b_PwIkZ!Gw<_`6pgD=LmX61 z4Arpi%Rj>Zps*1TicNE$V#=aV?}Vc8Zf$tbvXU>vTk<2$$7ch_?tzTAJf5G?!VEoI zGW3mO4>FYiiR}UsygNKz0-cJAoeHG6lmj!3r5)~4M(UKN;x<<5SE%h|Ozpp)?5b0z zVG4$Ze6T^Xl0WZ13HT_D1A|?INZXBo{sTEP0%>CLt3FIaNs;tDU4mf?je z!H2&J+zlhF1WrGJ&)=@#- zO@@61N3nS+CRYL<8FV91RTQ%@iC71C8T{;cjj^nY;Q1A-DDBaB6@FiT8QRg_v7>;* z-XF}pxD^Xzfpu<>O7KzW*t{W zY`@+8G^kI?r}Pra3-pfsY`g{Y2yYvf!v*&J6kg?#n@`jVgot_8U4A5?g``nw5-*fh z#e6H0Y3oSfh_z3$Zlt;aF>LR@$68kt-++8c!_bpSs@o^-dnW_-b~55YM+aL3xg37m zJQvV0Dq>y@dXdM%&vt7bgFs^G^MzkDHx5 z{q5w2rGV>0Ug1q{fl%LMs_=@EP9%x9_C-I&=aqH0T|OI{Y7Y+R$eG7|lR!ssF9Z4Q z@l=t(mK`0R;ml(!rALP3m9VLB3Cz! zZaVr<0tmvP;hhny{G=-S4n+C}7>14f#KCq`0H+w=wYW!)gB_lqZf^{b2?Vj0iD#9? z?-VAY-z+tjUk3x$JHac0NKN$*<{iJ4XU(p-d%R*ORneXPGFzn>V<~XLCbpvV<7yv> z5?G5rbun<~B8$>VTYTrH;9~vg#UnarAC%7^>fgLah-jB`wMM-cJo=H{N^bnHN@UI$ zS!94=T0}zOdO>2)bp ztKrOdqaRR|s{Lvl9Gbl&8%u@??0P*dif{k#1qb}kVW#OSwf0%!9|cD@@88@|8b)6F z4%p>E%ulsFucx9drt>B$`#wkI@$I@+DLL2oHLXVPZ?cSx=(#&j^t@CQ;9=a0<#dB! z;mN-G_+~j(-9}lhwY`Tf2fzNUSc%r@}ok~nFm$W}A++PH^ZszD3IV!1D+$Q%?`X1wD=k-)2YBq|1N{5{_fA2Zp(@-)$#CKD+_Pq zUN9W`5&o1VXW5Qrw_8`?09SR~MOCavNQJcxQ?w1c1!|(kd$w%%5dwFo+I!a@4m;Mj z!_hsaMIZ8)^4>U2WRq;xGdFvByB=P?U#Dr|Zus-=Om!E;G!io+V1&Lq>znawJwjG%O$df9E6O%r9iev8>2`y!6;g7q%4q<{IE z`~p0B2u!rWz+9Vpr;4>?w|8J>uyc zO$MMPZn$UOdn5(+M3NnQ|wNtG@<#$!VTWq??X|RB(zE|7=$X^$dOWB^uJ_ z;%W|`Z79N|h_b;Y+1KFnHNcuG{~<PItjY}y!Pv4>Ii^<2ZBZSwA&Ksp5zsHk=B|8pz97pkOF+i;!P?|?> z#yAHuaRt}o04}$(_+46%lwb*|v59SGRFS|fAqQL$^O{8Cf1MEeDx(QkmKb={bI!YG zPer0<3pfkWoqn26*JX9~7B7;ZFr9eWQR7)%3zXl2raq*n64 z>6i-cBbx2UJR3<>@mYUe%%_W%#eCXe{a)L!!_kA?Wt-4Vmtc-`^{6HbL!;LlGx=I^ zyy|!b{@`;5=uv)!K40OL&=Z71Ph>cIZ=fZsT?w z$3Cj#=}hrxO^5ZorA=~>XMaJc(^bNs)k^DNbfSl$S^|spNuqZwQ-7290s#kUl_1gh z(b{Eh;2%?~cvm+mM6#_o5mQ>*+6+;&)%m$P!UaDnPpHI%W{bz>MIv&-jvHw;|}OP5OOy9oJV{b?zS zbL*fh2EMmhSajh%YSrZ5rT|~%m%eZ+NupByZ?BHi;baEnKm{* zgQD$jB>YfCjNdmy3$|@+mu5Z1v@I@8jWN97rRibHVn93X1j|>6Ab1}3d`fdeux@W( z!56Ih$~!B&H!TRuNsY23{~7o^J`ryy;$wI_z~+D?)2z31)B&eVJG3G24{!aq!hrx6 z_|bTZ0*Sj-6864)6u>YeY9WS<@8z-A-$95vUY^jxQ+@#&eQA@*N59eiroNwJ`Yn|jUD{(gPDKLC_uAW9QWXBSY zp8xli;uLa(@JS?M9*Gn0mzK%mT@d_uYa@68zVtmNfpC^h?~R(n;_z5oGlT?VS?kRv zPe@VfdD?86m$>>F4ZeRhzLm)j>PQ}lci=D4Ao+2R=~T?B_i1GWdB_YMa23XzNu^I| zUhqWHyOy)<9XM<{wrF%}RBSrMHLdEk#5PQriZ)MD3->NJE23#%S(!MM^m;E)l-xOy z{YLL3`Ms6OFB+~B`=vnepW@~d23*sOxNhhqj6J+I!js(5t8>fU^QJl0Z}xNS7`v6# z=VA5fc1!whq{%|5wK6fM%5Gh?>Y8aI}JKljfZ-~w0TMKw%XwyJ2Zhzy!G_ts*^@x$|azjD(6E z62)cCs`A*w4zR@UIci(E@r1P+6{l<6qCTzuPilRGu-lEq8|A1LQAkY20J6ygLp%7wtb@7b!_ zwDKii8=6(d>81Ykw|vTvnwA0C!NSPj?x&z92*WoK(tKLYbgirt)to>xQS7OhJscPi z(!H1c+u!#K7NYab2vTd+1eF0sRf(&BNwLG*G^0!h*rMHxI)XEpvJQfI<MA<>6CsoNkC)uo2hfMH- zJ-2kZ^Ek_27CIM2K1%#pO%zkJUWt(lk3~LV;LMBjk~gG@yA`l2a1<-RLA)iXHoQG< za0|Dp~XBf0wi`Q8>8+oK9h?Tm}DW9r< zHEPM8F#W;1*z`X+ZsI90doj5Qu4yZ2_*hATnpqn>(?W-{p42M&kUfX)CdT>4w8d!6 z`FGq#;HQtUB6$ydU+_}4m>j~)eD6H9apH~@M9@{$#Phl_<%`Vmc@2L0N@3h#^EF}6^<)6)$+8Z7L*q-b0nqb- z{B*hx$9m}RhfjQ8^^61`ad}e1x&*~rxsrPG9z-#TC#Nmg!!r!*6b&=%jPo!E{wOCj z_Ut$Bm5&NeE6;)XZXOa^b@xQoy~#&K|8sDtu+|ff!^BOHWfeiICYD%C`#2-OnbKu~ zk%>a`qr52W=ONExRyMABTe!SP=imEdGRQyl|1oaQlBj4zW<_`7Cl6>->pj)x`zxpu zijx+Xct$&GI;zpJdqsIFO=d_p$@lpxJgFESJgI=|$##ZZm`nn=wi&- z5O}6mjkeY!#L$Nl1v6Egh4vyJg`-7&*Ts8qMZOW4HIzATTYNU=^DX`C#w8AfliUxGese8_%)j9x$vHgCt6o>PbvN(?+!|uWDKhMb zOb31K+rM5+*3}e1f+l9j8v{BzgOIrw8Wj)|a>i!)t0|8b3kA=_iKSy-5QhkQ>6}hp zFVZQX3eyjz!i;^Mwq?tX*e{LBMTflLt*&www-y(sGZ3=CsDm7f!@QS(+HElt9*Iqv z2cSX~BU=7&0BQ9Kf4S|8P_7Nu=JDw6*pKru(X;Hbow5~&Q2BIU9q*b%|47j5aL=`l zE~v3L2YtNt7)Sil zbm>Ibz-Aj07RP?}-ushJI6yzAlcqp?3#ZX`UJlG66q@+)FTe#ft(;X_@d)iTPXl#; z+2Qj+XWB0lN!q+o~ALZ0J?3-+#R-Qh6E zDDo_QXLQiu(&A;+vRMr6#sx!6{qJ-?26kJZ(gt`Sz7X|jkNdul@;u1iBwLm?Ez!cM zj^<=FA6g_yO+?k^+|p84+6uzFt7AG7%Q}=oPo(P?U+Otu;GmDJnavvKE(33C0{5-7 z!a*bwxgkGk05}qHTJm=M@0nNhT4fum6MaY3=X@0oTUt#CrU|sA%ouS(s@;UJb4(-6 zaz;fYxS-)%x>2k}hfK9h?Z!*sSDe!@E7YqScAP+lZGo{od@SZis}umtiDM5I8I8{%^Q zm!O5bO833G<9X`UytvIk!^f7uq$<=6==biu{Hs0rbv0dV_u*elnQi*fNS0;t>YRUN zUG#lC;o#BJ*0(yat>3AGU5Z>IU2-9z+sEdDIdP1%etEj}n?+7Ed0KQaAJ{SjobGu} z^Smos@;AKRzC*Y|%-XN#(cJUKtIxFUFI8P?Jl3wXeYvpZDUJn_Qi-Qj(x01y4H6Pp za*Vsh`H9ZdFScHW>w9q(M(V!?aA_G@r(35abYFV82o2_suRZGYfIe8-vjLxtv^uLRDx4^wGUrSj^Yx_#Hnu`a4r2t`r|Rk8f3MBJsVw*; z{ArE>UZ*X5X*Ge23x7T;52D4o%A7D)vWAp>T_uLLhXQS?R}_9Ia5-x$nYTvJmCYV9 zzZO+`pg5*BQiri}b>5|g*)T8BM(tII2)7|GmCO@kv*;S*rexoL#)7HCwResW5B;IpK05pgFStU~( z?YMX~dK^1#wIQMCvg!$^@D<#!t|+_ca=pUwO5#pzh6>~R; zKohJo4ng7t{g5jJJUL$kJ_GEKrz8aH(VDrMZk-Am!8ZjqxZm0nwn6R)1HVT2p02+{ zUCmwnpdKqwD1w)a7!IKC7D3<$z%7J8r?e7lvJuAg(tqLovGU zp0~H(roo=@gdc;|fgQmUBWZ5!I`ji(;AaqW!BvSk7Z~@)SgDu}4C0-tAT2J$#0om6 zw42G{)mMuAOpV0NW!1)WYmg}RLU!7_^XFGeIp!aA^9@w`i}9Df}JwiKfOPNR}Uu2^6v=EGmfc+yHmPUiN>6d ztzxC>p7;CSYlgCl^a}I#d5-QFqIzUA^azn~n9MN#$?U4536oc4bSb`X8&=THl!Vo( zB@K&5y7qjr!v7Nb^$NYb{wg$nX@;SbSIOhAhp~?1X{_gjZIbr@=?BQo_lAUNPypL( znG9UjuhWt6>A-Aa(~k(2dZcdRSRbzt<|vIn;VeLnV=Q(*D9>}P#DA7^g_u%ZuBtL9AL-k zuj?!N+gN6m6#)kpQ?=CBOL2yW=Ys!{_i}BtT4Ufq2G}%x&N!2hM%%M!WB%t{tOOr|t9;!}Fs$_`eN? z@Ez04P!d!W=>L{B0-St_TCKZrr7NBcUF16rxd8GuM*(Aw-?F$EYWE>XaKzXGOT0_PS6 zi7n`OR}LqPc2+f-+|YiPe}& zr8;Ix^3SIw0p@=`t|`0q4FgW%gm;2l-AN8`dN~?QMlgRRjGDY>UM2KPwP79XcUYkM z^PppIMLoy+#pTAW?~9~@~4;8{tYtOI}RDD=$_wj%Yc6*V2z z^$9%^jgk>4IrULaGE#ur;ZIS$aj)a>tBzGlWpF`>ja5rbXeADabtfG^Ovopo5HBSDDMsCN-HbM@7EG&o6hM`V@Js8EGu$$q&qINb|)pgugBd(3% zSLl=25<4YrCxyx+mYh%CGd>k4(TaHCGMuO9JPfL|!@b;#A9Z)Y$%(alBFS%@o`)1VmkU|o4aeKAl0CRlpKdcg;!VVBTM>Pm-(3LL^g zdmx(u412@I3mHs9WDkhGY;uvOb!kX1YH3Rm#$UQI`T-$wNJkFaOAG?hEYy0n0!%zh8*J+nwl>4{Bcgl-7TBWj9wj zV7%L7|6|ZMV6>EVL*02iN7@wlp&{Zn(6nG5QM-Xx4V;*l^q1(39?##vXUiePDsPn2 zj^2cglEL~i5#p30Bq%L_;7MfkM$cyHGrK^C8YQcsIxA+N#wsxzJmvr|f`V9QK@>Ps zX8hYBC*PkE{sUF6x#dqmpNGr}{!5+}{V&SDmOB10wYsXRHtwATOI+=|#rO!Z!P!Ph zFUWCxsGw${i7%5siC=)+Pm1Xy%77FjcoG@C(G%%rkeo!#QMN#eSF9Tv6Hwr4Smi*`IZ@CpK%`W0WgZtrz%l10T%e8Mz!m$&eYs>}Xzm(r4;k03(XnbYW4Y1vg)Y2xBbL@+&>@ z)L)eVu}W-M+)Zu+AxXojuVJHP9n{jQJGR5srw3U&=7l_v2zjuv;Z-Q3ZxCV_EId=Z z6c+IFPrm^G0(|l1M#*|?+C+OP5OSoid+d?0Tz&x-o?A^H3Bv1&R1}#%geUVc|u4!)S9LHV|L-K^Md$+_wbFv!mO8SEkR-RpC2~@47E|t^Iy!laLqWk#$LK9Bp6!v(#)+a@ulbh>mA@+2a3#M#*B>C}HD| zp^Xx+QNpr@RN<*~_)&ZMCt>AXxl}18dMJv({?9AzR;-0a324%7PyeIbu|5U*%8%uM zQ*n?#J7P2Tzk+_j|Dyjvb|mt@C67POr?gS>&Z4ERRvIN6+=2ajyMcXsM)_7w{u0*Q z>kpH2QTVADvY&!}!T;u6|2Aj;<6Z!jwfLcDskgBDXr_VnB+%NzfYodP#bK9)UA8Z9XxcfJNdJp)T)S@ z`20vm*Od3KV*OLBe_GP|=l=l!0RR6JwRbE406+jqL_t*Soe6+nMUnq2lgVTzCnO}C z0Rn_8isG?CfPmbv?#k){;thBJ-Uo`P>w+kth^T-fUU(uZg06`7UqFJ800{yKHzXwF zPRO0i@&A6Ry5H~pW+sr6kjd*zey?A5byatDcXfBY>h4}0a^{XlFEf86f0u9q%5e_R z?a%zhOBJ5+rDRImp)iHI;6ZFJxq41RYC9*&%IQap?c^V;dSUqxfy?&z3 z8d5M(4WN|kTK_>KBiP*O=PDNaz0B~qO*~mO@bEIZdPEx(9q2pVHS7P8vr@i2Jyc#8`BTwP^_AOA#eXrRYJVHM|HX7hR3t+f_c{=` zh?92YDFQ_@Rq_V8LTzlQ`A3K3b7Sw`_RO65_T!`%|Qrrzh~A{!U+>aQ|aAcj031&-;!Vq4`VSG}ooiCpN0}ZEX4Dw7iv5 zGr2!+9Qy5&ZOEB^eVgpIf8J}qzUj}_SN@KUHjzfYY09Wdj#4-18S5UOOT!a6fqAFo zxqSIbJMz#mcI*)c*@_jbT!vQjFK1H-uGElX{R3BncM+T!nge+%EO)pB+>_ehmQ?;; z5;R_3s{-VL5FcCmQdTO=l8TDq&!oQ)zmZ;PBR@$uwv{+v$D1hnZg{kE#kt!ag(nMIG z&Dry!Jy-c1(2*d_SmXAxR!!V)o$!FIlA@$vQjisyvk zi!f2mQPs*18WH88tyhz^O*z+EpSfLpYEl9bfj1`Qir@?S^1#;MKN;aKY7iiSmiQlN zJh8%otgrtFOQ{`KK}hhy22FB%FnwwS72PT+?B)%8oo)BB_gUXQjm`@u_zkYyib*`m z5?6Q$zC&@jPT+ZNGO(H6#IL*T^=H(7PFD{QbqAga4?Jq~7A$cT2fchRYtcl5iY?lu zaCw&r1YYMGfhaDt58(EVQ{mOe^^qU4Eb zv-Q?)qeg6JyYIZMQmDW@R+8%zFnrpVZ_TW4)j&J1|2H)KLQ&FXIYlR@_pHWsfvu=$I$;PIXc4lTIoZF(ZK2=;=w zC%`;ffwABdh+tSu5?-UFKu-%4i%nhuP@v=G8Thh?D1_<2e$Cd@*jr!mC0zKb1Y97) z%H2EFl`Xw1W+MD;Lc|Glh#Unxp(MT3af{JSCPGvN1< zLWr*zi%`c|Unrlx^pRGDIDM^Oe?jyGM?-i~n@w!W7As>~A$Ln)L zkN;)-iAc!(oR5mW*y0sY@n0O+EZbj9VA6-4#XesdJ!)s`uYS(FsTXr7%GLwNe>VYV zCMjWa-HkWfz4tvRMTsm*m`}MR(-@)h^WVEVoJe917*r@O3iQLgXywXQ8zm*vJ5GAN zHTry0d}IBff6+Doq{~#}Pc{B-Ve=17(&z;g-fpGQrliyv=BX5ijuJUpj-(Uc>T|B>&ky7 z-^t22G|Y}wqEX>kl<>dJ8m;4jPg?t&J2kUpVN6r-no9g9?{o--Dnn@Q;D4zK>iF+N zQqH`dQU9RL^`9dZMzH>8{`*2A<&j)ucJxI0iNn4HYi|F4x9#>j%bNQ1(SQa+#BKd= z;Hzg^MEQ0oZx9NDaoxCj&I+JYK&My7Kp5>0I(H}2RTNfZWKlAprP*5ocyin94u>Ps zPnWn(0i9kw8DVULgZ{jQOJuS>OIwf{Y=>b(tg%rRHmPyc+8=pQDjH+_)wMt8O(vvg zw7-CVq2*8T4Zc+LYyF>Urc3qd2Ur}nt!lNiKKe-;H}0RVaK$R`|NZedw%xYFrI1~zZ8&KGuz}hii?Sxk ztl4v)wTGX2+Q-@b_T1Ta+HM z_`kWgzuZ5@%OZ9@7O|2mqxafbbNy@~O+(M~Pg;G?8GjfY86TOSt(5i0^ck~!`wZg= z^QtDTcOHEBG5hEENim;HmEEfK$IC|VYcJn#FKd$(2@}tV>Sepg3a+JZv*xC)J}-k` zJ#qY{TpMHjDdeu#OU!0^YWwq^#2_Me!LUdxg3@CK69b<&*Mu4PSPX)?ZWiGZffQ=s9?6eaRq25NtR|kSp%5V8 z)1O}YX)8gqZ*TC#O4F$<(CbphOryKFZomOcI3gv)MJEd=P1bH3|A2%V79~xxDA|Y1 z2_(v#(>j_Gu6Qr!B|&r(lyNXQ>n(xF)5DmULrW7$CX?;j$JlBbOr}UtGV3mFu$Dy$ zHRJpbA(HbQSrIYMr^*AKGOvniAf0g3@n2;li6fCerzKD>C3U|;Z2|Oi8*NqH%pY)^ z4SVTX)<#zXH4$v+;h}_MCsStUtKD%zoPxCKJc^kX6}N; z^7b;e-q2t8Qi0MYTsOqJ35uGg9ZT&Qo+$XU)c!JWqgc@Q;^~pvn<8Fl`J>`h^eg_m<%mm2waI16m#f{I zwe)Mry7K6fF8`u+ca^_Y`+vUvkHsioP*+MB^1hFpZFh~o+m1QRwS!b7p@8sQgL`m?9EKUKRAJ3YDnT~U&?vLSvDNVp`(Tt4{^eZgH8akbF;5P;?{ zmPLsGB27-Ih-fu|-X{!#QM6klK+`*?lL0soI3f-H>FHmU6$jS%E(bi-6qBBmsG!fb zlm0DD)-1tMo)(b`w=v2enB&*2J5&_TD7lTTH&T=|%S6t%-jE|ESrT*!Iz1ClDWvlx z%1dO5@$yG#eF!jKCZ2FbU%Pl`QQEGFBqla|0c-1!V14LQ)-mf&>($uo&^7*pGX!hq zJ7YdXsX|H}IbXAABk_6awXn!RVkSllbUZV5KXVDUsn~`SY(5E~UsgIN-13BYK&Ocbd7-uAf?&>nXBZdOw z5-hLIli*z`Ch^h%2$n1RBcuqf=!=gP|4Xf+@NUlSkGU=9Xgv7%G@GgqEq$B&+CKWw z(nl7$d=%tDhv*lnxSQel6N0>}{PT}Y2pr_9k2kTp0UpcoW?9vYyK9pD_78s%kB}e8 zPOU>E8(%y2HFmTPZp9KM#sj!S|In|G3Bv2LR$)o*u8I83IFMhS%-N1yO0Ea-pHi*K z`V*cOt}1`9!<(A;qNg`uG05AFk9C<_M9TjnXn(FKDKev6=X`lqDa;zW+!Q>CUbyu} z{mjc)8=xG!(||j^2m*g5>XV_AhVX<82BxWobCuwNsot42CkrG|0to9p(?_86mjG?n zIS|EUF{9VBpuYjj58BAT=S1E(YvLAdz2Pudls04YVXp9k0&YP1KKLbphab~pq-nAB zhFfoBQPMv79P2gbF6-#il-eT#F4d;FX&229@%{u3vc_@Cn=kce=O zT{L`436an%Iz!K=53eXLbuqi;INSE6XIPWYta0Utz6!)-UMx}lg`xoVh)>Sk%*Fbr z^Hf2>{}i_x_%f`sUf>fRj9YI&aL~)7C`nsyiauITuFG2)r}rly)_}fP;C*}2BTw3l z+4EK22HR)kj=rFz&C)N{xBqWy`BTx)4S`k?W6TRB|53uBL_ishg_O59y(6<|r#>m< zE1EVUx{PBXox<5F2|VZLG{{50&HV+%Hf__zmSD0HBV?@4UAbt*yMQaf1=A*LgTaFa z+8_V&cYE7u@3E1)>}G$y>DO9JOHrcvd_%8BDX&EX>#>Af$y}UrdjWkh@{Rn}c%t%< zXNC^aK}Ua{0Og2laVWR`@k&~X?FtgTxSU~(V4G@EhS29g0VL^2!1xj8dXbvK!VB^u z2<3}|(p``1QNpFXL%*GP(RUt;J}!R{h;l?&$T$)U4|$Z=9ppS0Yd-!{Q0TAHSJN{; z@^QOcN~y12a*-W==)typ#R|7rL+&~+JQGeLh^`0ykX0%-i!*U>(15;DdW(~kg);Gz zTwU=9bC>`uyoA@1@b;Xp4rkp<*JNFy3!>C{u)&zY~D?MZuG6}V)cXp3)?B2$aVa41g6J3w~|6J>jO+g>4 zkY-t}kGpfC{YHus#z5E!zt~oD_#p?{v9BFtD|9Z&hLpeFf)V24av{sJoDJ!+Mex5W zf6uD_UYay{vvxUB5+?DE1i`7{hm6UO2-a@|ca7wX5;Ty$zyl6{fk*J_JR%+wF0jy< z9F2ZbM=OZ9ht3yKoWKk=f6qQ%Ydte2#-gdIqqXVFeZT&Fte=ztkcgRBOkGm02u;p2 zQS#%>c}rPLu{;7gr70&Z!7$rKXfr{(b^L=;l(f1KJ7oX8VcaovH95f&B>2x;sr*I# zMR*wvshKn*s>znq5PYrv38ReUx#!dOzM%59NwYt@zZ4}eIa9YYa7IZiz8V|2nTbV( z_@%n@{q%bJf&*6Re|FB!+-Bk>N7bIr(sE1Bfa-tpqM(@|i;{WTdISGCqr?>@EX1gn zb!mS!>fT6-{_o*6`g|~Fp*88@r|g;8^PC=AZw3wM=U-!M{US3}~L{L5A>kL?#~i?lxu`EJ%N39T#JZTU)WT|!Q^Yx`M?%$c>FLKYvr zwIz(UNOw?>7Tv3K*2M~S4HOO`XIY@P$a0ZuVl_{71q61H#wU-hIMKE%Jrp+-$|N6@H*J<$**W<-4U{lkyXa zbq*)RA$L~68}W^^gRq9gxwux^m8(|7b&^Qg6ZJ-Lh*n2$$scKs;ANY>VzwgJ zgdts_#+xg>OPkm$}Qc;{S`e{jn{XL-5;lNx?$h zY$eG{*NZmqbJ>1^!F01*Uq}vwS{x@{}YtF$eQUiJFBlM-t&f_`Q_ zZ`4%1qF5y7=>YwZ$z9ESmN-FrF609sXCisy5jctolY1NysU1I)l0XI?1yd>}T?!x6 zzpo#*#x@-Ps^#$sM!-%Z2oYH4RufzY^IR`(A}7JjS0@xDaYhNZ-f%szb3^$RhgL&q zHNnqCGA3v~T4>S>4IzO)9CF7Npbl=m>5$^+p-;-9WP&ELeKov@kKzmTaO4NY2-6AT zV5+kn-s#n2NZ>Kx&z|(;p|JdARj+}Q%JISyQo{?TErEWtf3-^u?6U_PW5f4NiV`hw z8fCVPxv29(zkJPxug?F(Yq+*HE?0}E;RP4fCfxA_cKrS6Ff^I@XGMt<_WCctBkfxL zJ{NtqGw@}5;zLi`Oe{(?YNIF_Jdmw7%A4LJeeUBa@7~z(Q;-}dNPpFjmrvmuC^n`1 zspxN7{v-cUQn%{M+120wkv;P0V|K>7PqWec?5TtAmTNKJ*CyP1pI!aEAK5Ej_ELMt zTTfP#YH;Pkm%sWAdvfYDJLwItw|gc|vb!eS<7tKuA8zk_%gJ`&zrD(qNijhK<%}V= z&|LScU)x{)-)-72|EyaSj2yAEop{oJ*?yz<(II(F_V-)wu&aOYW1Bc>k_}KBf93u! zvmJNX!Ond5sWxoL5c|!|f3RO%|64m==V~x!-uU~QZJ}_D-g_@Q{T*+%T}OiqmJzaQWZ2Z$Ep< zOZKu;-*$>^zumS{fUI=IJj|2 zNL%W}(;au%-cEkw>s99cT?y=cTm9G-!rpZmU%2wJQHt8wy23m!B4>DXLQNvzOI)LIgWuqLlR{py575(5{ z#sAcXqILLhT|=ogvDF0E0G`KJm!22g2I4<+Q~p4=Lh^T!l)uQDaFQ+ZZ;9hSYmR1( z6L;J-(Qc6J_Zt*pig?A=|051Pz>dbEM9THD{9Rx9=M{m__WG^nU)B6?)6V}k82>e} zhv?$B5MCi>ieEr{&{sdpeV~Z!-ow@#oTDX?e33T%{^Ae`br%{g0?`fo1Sn^Y(3qQ~ zjEI2a4MM^g3oHMm2YxNI_)^%^rJ>oot|Ct)+zouUO@{Jm^k_w zCB>~bahR)5tUUd5Fm^5J=fMNCEJVGr=t($=BER)UTKH9M`r7xQ6eX^_>cj`H zE$FAl;XV%}zk172MJ0G)iXX2pRw`PtbUR@LvrrWzGQNbNC(;j|s9j1&IlJF6vM714 zz9MU4Ev-L?xi+}Twx;u9OrsVZSOK}U32hs;7uxJYgCsop<5)mz2wBf6A$Qav7oI-=@|{{#5kg9$Xh&HzEHy2dHnKzBYIMJbU;1 zKVp*~oNSj~a-qHcxMOU-6c9rO54LN5daZrta~Igbha6;=Uvjaw!Rb6JO)_75+zIyd zw5M(Bt+$fGV6G1-91z&7^LqyN?{Ak}e1RQy@By}zEgkxz`Pt83Y`?tzM!~h(z(Ip# zg|X6>$cmv^efuqMKFKcn-*fE3ORunRfAC@Q9 zh7KKSS6udG+yCXGZOPK*vcze!3%~qz`~DAq?8@MQ0|vUe_u{3C#iu5F;~W0dE;|3S zQnak__K~l!_x`4Z&4L9BZOX$ByONZ)6Fo8YVQ*6%s?%oMYt(MahvbA5|5)`f9%-11 zF<<#VONx>SQj}bN>6h&YDS8$yTUuOuxv}~`^l9!}>B9^bFIm31{82qoif)~Elx|~` zRFQSEX*F=&1lQ2Q#fpB#|85!yLA8d`72rkP{?x9?J2Ftp(Z$PFh1~E0rDAQu@yFS^ z3Fx!+kuyqeAAhg?_V<4iPhe5Be#Vp|wDkr>30DG#qC`2ZlQAK6*LD3>&40X6Mz`7o z^ShXh^;qw~_~iUS zhfeP#h(CoWOpOk3mi<5lLc;x}_(37SmnG;=pEb{(mEvamZML$JJ8i3pEM~pTr*uVN z!d)cHo+7V7B?spx&-dV8%2rVdLy2uyoZ=1C(3yIMX?kg{EzZ0stEUdmwtR& z#^g1y6nrQ`$dizJ^`~@O0{v*avM^E`o!Rf$q$p|f`eJg;;v_6RwEtgUSG_J_Qi)NS zrzLjRyr~X7fC}p3QffWOOKiQ-tv3sG>y2Rj)*F3k_uBChwhMxyPP@h^WG(2o3b;|_ znk0*o8B&xmFz>yqEJ_A(JB3<@_u*)3-fq^g|3?bt#T%5jMDGJ$z?^C2(>dXMO_}yb z(W*g^{{%~dXs^#Fe=7Q^;ca66Bj*~m<#K_n8dx|_o-)P0bm95-@5dZr&px}v1`Qlw zKe_IDJMV%EZOoVh?6Ql$ASFg$E$AEU*b`2c6~;3@@jm&#-)OIT#eOz()@=L!H9xT# z&&;%!?e`M<*)>nYnAP{-OfJ$xpQrxlr^Jv+gTPRgYA?5bD{nE*S}HQZ?mJ1I^6#A zb;sGvXJ*+Wk3S(xn0H9|(rPEYdBo{`42yeUIJj z`%=0L8M3u*d26x>lP20XzWH4{c+6{T&pmc`1j_+|QlwRyy>f)RMgap}0kE_E)`KAxn`luX&Zm_Qk&4iEf>h zyU;%KL!Yp_@4d&qA-E%xMTwsq0w9j=tcNRu-4|iecU=$=oBy2t^Y#Bc?rx>qb;!An zO!s*PA|~f8Q4RWmUUWLS^&I~TY?*mQ03I)OMW6bw_+Rv|`X(agc-!f=0p%7{m!x>f zU59{flW)NGCscmBjpj`B`Gu0XY~?KoZ{qQ1Q_VjlO8tuDak7HE@#eq!+|osgSc9VE z@Uc>q=rGq6QYhzjg4dtd9sJ9cD1ZE|k^FNU{sRuVEtLOTdi(2f<=5m#Jt8$~`bP44 zBiB!TPXNMFe%Y9kVHyoAuR=GfI?G&rf{07mf|W|CrE1A0D}qgr{EpYG4g0ZTd3W#`VB#y_u~^ zu)ezbA+7~|IP1BKEO}wzH(NpU0-O`jvq&Sqze0<8ibZ(Q!h8TzqFmpOp!5#D0$0za z@9YyVP@K)~udO$uwe^N`1XTD29XO1l1Q%aPy@mU`vkwMkKINy!mVY85N9Sh&Q3{}j zsZ*{8o;P0>CE9v}q6CW)2&EpDlBR??eSQ{*vnL{QbbjX8BL52D90k4_qbQj^ONtU% z)$F~y6eR-&xMGi{;*G+Qta<;hZ-2ajh8C?yU550M+oOAHk?SLWD*EOAzv=kTK)@M4 zY=t@N!=JRtlOMD%UwDBXf9w&qXwhPA@fzS;OwRlKMRw4E2g=&y0$E=)`8grSpKy}4 z>O5+%d)@K&wJ&|atrF>@-~G{b_Bkn5*n;zetFN%pd+lL=yLFts`Erh(qN{MRX5qY)RT}$-Z8gy0{6`u4*#CUNej-bn54``q_R;s9rLr~H z&!i~%@Y$cTkvco&s>?66SGzkc-( z?cz%=l~v3x_UoVj$Ofw4J}`N*6eXv+Qs$=XueIHF*)jT`a7~*zN85Y`X-*K9jdVuj z4*>0qi&D`3{kC!T^>2Mg)-ti}u$*z-3nnGfDjOvJe&WOLmsQE`en@M+I-Ti^jtKFy zP|p0&$L(I-dc)S6BM;LVB|5(VYrT!t|9vW@%wm~2pkK4|9=QV#eav(&=nZ*;aM3^j zY%!)P%FkMfD39}=!+=4U{uQtpG0)`ihSDd-f4w-xJpUrj^C9|f? zA9`3s;Fim(Zs`i0CnenQ&Ete8w_ytFSx))(L{3!cgPf|S78!X@?_Ev$S>Po38?5LV<=@o&uju!z{#Uao$^9cg zLyC0!8z8F(BxXU{dgBrZiJk;sNGi~VWI~|gb14Z-M(atE7?q!~gsA)*X@o@)z6TKi z0)cfnh9|5z+U>q6PiRlG%yqR09sA0CB`b(yAV`7}KoX_&DMN&I>jE?<=nB#~#s@eJ zFdR@9hq)$;5}i>J69KwaE&uCF->)RpXyFIqDKoiP0IoBGv^#+R) z28o<`(kY?mO!WtQvkQ>;b<^lh={%+ck8qCq7Y2`vw7hg^#ul}pxSY}S?5Xtg!Frro zGmBeqwDranC91RD+KbPlP#)Bc!e_|_YJVW)ol zdSDbK<$9~3k19;RrQpI_!=8gWht%FVbc~74v}fkpW7B5(R)XDj-qshZT>Yod&D@T< z*8ShxvXfWgigJKRfwZsG*fy5@sp!LK+@|F}mW6#}l{HU_lK07?V6t8L^{?0){^J-OHayq2 zvpn+H6ZY=2J}BkJGW*I!7uYdJ9AfuOn(T^_rAwFCt6up^JNTe~lVwIf+jiJ6ZO=K- zEiKr}vX!>D{8?w1oPNd!Y?qyPvESYBbID>|->!bJ&5%L%>GQs1-~GWgc9JYuEOt9F0>#0MN6#ZEu(Lve__*yK|FBCg`hw19`EOa&%(0={Y;AXq zpJZ=($Emj6c01S~Zv43pgKbvaY=ync=FXWT%bJ%duVZY0u0>&}%? zL_Fhs8L|f9d5s+B0_Uui6V&-*;VT@fXPWAi}1P>HF^%RUd>VFf_C+BfJux`Ei{h$3~ffr8Y z^}}4TC^;-{z3~rn6ubwOe^c|nqEAP{^^E=}BLbvI+3|`1%E8HBQbc%kf*>|rIbAQ~ z)*E*jRRV-ud|eQH5hsslL(e=b>Owz}AyND#O>ufrdOrgr6^91{6O+)1I{$#L_iT4L z_!WCAxnQL$iE&=(>s+}UL=dmAwJd)bC1oZrER1g%|9}J)LQIFbzEWT861gm{gYSu^ zBUyX;UiK)Z@G}^Ih^Nj;5g)k1knJ~ZrrTAf=+>LLblHV#cv7pVG~V?Lw0Ak9BadMJurLynZ)ILLl@-7lml zxmep%4z$Z;QNq?6Zo)ZEijoI)c`^-BWT6xR{cY;gPup2iC`_9^-M;e0 zFW50hkCoD7g)B{eYFB^f`?h595?PNdbGln^y|wM9Z8#tQ(EDtkJ@@c!Eq}Z9Hhbsk zXUPgf_VO$JIJl>J!_nClmrn#o4oOCFn;l;IcF0BX0KqCgAkSLGuQNZa z>w$Gf$=7935{J2x4;F@^1HF`92@3k_Gyh;b)Zpsppe&K~(1VXmRrWe$Wz$b*sx*3W zJo{u*3Z5n@ghuW#TxYr@r8E%aDQC>{sH=`;5%lQPS-P!B{31^5a+YLB!S(Ia+jiY? zTWwvEl_>cD&T-+Fb6h$TM_=6^e|nbA##t%85s~0Ljlje|&d(XS!!T>fxFF`*DDaTP zf0Un<_V~0}wn9oNNHK4q1~XxlM&O`~eRTr|QF$0C@HlucTdMp#{`AcF!tEI&0!RMD z#nuk%Cnemj;L-;#%2RjdnIjMIrOUuI%dLoM3uWizH14v+YvfMDysdi1yaYfxgdR)~ zd)XrO>!+p*m)cV?{+9U9B#5)`Ms7b$eTsY)6U!Re#QQ~3Qa+)4F%JwzBwE2d=RLQ* z?JVB2^)Kp-N)lXVoRU)i3-uxL$r-b~e+Ii~X*K@C`@!1&waZS!eO{64KT&j)YK1Ce zgO9sU30J;0+j;JMpa}GpHl4XMz;@YT8}G-t{tH}yrE}_J*P>p zTgvz^Hv1Z5-1vz)5L}13(w<4$aTH$fEoVA{`o#u;RFNf^NOhpv%>XW2LlV zKx1+0(!V}=-P>v?2&O1R4IX}|e9&{Jcjt$K5t411c(Jdutpi2LIZ~8hQKCK7KG-P* z%+9=}ZSbccLUVAufnT8o|IF~7`eC4- z)1t#%U;186vUK<#ZBh#~&dz}Y-gneUdD2Zo^RI})B_3s772)B@{D+s~vg7oaR6ovT zZ-25VdEjxs9@w|uyliiufTV2ZB(?Mfoef1FJY3_g>-}_-1^quSX7c1T+8-GgC>i&k z^g>2?k3WY2K9_$}$e)V-i-Z3tAo@xvz|AY~`|u~U_2wbF_>1S;>yA58XOt|`c9wzm zV-zKy*LfEQ9$;U)L|bfR&A`I;xc@puijoKH+gE;F7AQyPe31oO6!o{qpPcIFoy?p$ z)4n353)?!_=E4=<go82j1xuF`gl2K(#ZZ?|{8 z2Sv#)ZiRtzL)Or?jVwk!t$etm%jx;zfL*L{_&67wQcEqJ5A?~ z%$&QxhHO39Zogx^z2%+nmPN_-Qk48s3%nS2dW+{1C;rR+^tZp;H1S}b6jf8FPSxSK z({1$V(e~@>ex&}sN&~gzn$jSj*cR$rZ*()zWtV@;t#IOsN3|uD5yx~NWj_bRe)3}< zw0-y9)0XLw*51A2N_EB%BpnAko+)M4J@@L27Fm=Wp)*SSdSF?omBa;&6jAK-L$<8@ z`~xY=SdUfU0-c*O<C_11dlIt^&||-ni1P?V|$O>$>$WHa3~dyIS}t$jm`GPNe?~Y=f5~_8NxOH z$d;YJsX1yrk???Dyn>}%C|!2n zWqXfeEX(|N`K%Y}|A}dHZ1UsN+?pune+iG!WUS1Ub{jEV3Xbg@hm)iH!jW;J2z%_Q zXY8S;W`wdK<;k<>nSws~>Sd#L-PT5KH!S8xDvwV!^pmGSj6n}QHQgSYDoaNxqR0`v z?#zD*xu>?5?JPw?UOYM}&lJp13gv6c<4@ZY)8}};T&X8oY8LhU*Xv%p?r0;14|QB( zbjlZKXhX__!r(i~jV4#XDbNd;IR)zpGaEf>gzYeNYkd@B{*V{TL@82^tx@+q`lJ*R z3#_l|k$F0}C@0Q;RX#r6jNW5M+fD~YgRA&}LhblGQ)!g1NmCw|<=sN--6UG1>%@Pu zMLtH4+F6!XgS>5$PLN(Ulp@fF_jf<|n9ZKgfz_x#!j<`t#c6|-;Ct=7om=X)DXo`L zz_4RW9{;JMzOpjBW5Oi2DB<=*YLynu?QCqlIflbrrJ&66*O^TF_xdjw?u`I`dYhmk zK;u8@#oM_9L{!E9Ex!HrwEpK7B}Ln({TDBVhtTJA8It~jT;#cc>w&#PDJcMQA>{Cc zAojskuX*SJ1}MitBGLgXgb30CpMnvBF>R2a0Z0F!8>0#U@=5?R`Qkkn^qFpc7O65o5m4`;U$BRkvwNOhqOAs6%oL4I`B|6#k38^31}W&vy^!*! zqW|LHKeCRSqu7S>!H=G!tAOvd&z$pdd+$3>u^F<8;DGN7zH)_K@y)C3(6M9fGMz<& z61z{bD48salJoxOQ}&+I-YP}OLT!(c6~$fm+Uf6ozbq8{+SkAOC0%#BpJb}OQmfC< zMYTTAkFE?>Gya(lCjFew9pOfr?_7DAuKC@={`lA1>{Oj+vdgX`?JqZ7?`LdqKF1KY z-kh(kH{ZL)-uaF<+vh*?saV`=9OD3R;xD}9a=Y@X@7Q5u53y^m`i7JVAx9du*hJqB z{bf4TWR)xjrae8~PB{5(`m)&9e(>FI>dcWnZOQUg*1x%ropbJgbbO#Jfv&vlOZFNmg%;^H9qJfbzL*pf+O!xr zldU(hDEY=$zO1b`oKb>xjxZ?RWdVX%E{{$Aq8NBc7=I2c#6DvPa0i1g-OWFIn}gC( zrg=Incb3}B3RyUCKA9tQ91svj!R$DEhz(a8bU6$;uzIvFN2T28Et$B)9)5C00Ye6W z_i=$2<&Fj7PTOu{!-neo7hj-~b+{#`cgE;O(OV!Zsz;xi=^s(@#|keiaO9h@TWx8? z_Cr;_gO$`vPJzgraQP<}=i|a>7uzGk#b+3gbbjdvJSv2e`j_loq>LD{wHC$tIOBzb z#{!}ZB%w~=)#Fk~_?$zeh;df=Zzwjf@T)?WtAys?ER18_RWV0h4mz87W7fdRn*?>g)uUT;@N09m^mr zT?WhQf|K9CP>j`5ZZIFhLS*`^xi(eS7Q~Y&#|bX|bDUyhv*v}n?l{~A_3!5vr_^DL zTW~{=;u$w=P1Dx5r=|3AN5`k2prNol1=m;S{h*{8C@VPfMHz{L349cz)+pmsQg}_D zJx^gN-;S!re-v$O*BdG8l>W`+E9$Xm|6megtj4Czn5A+qkQ|_lwJhN$PqY;di{4FT z?5EqqGA^Rs!BD->o}U!1nR6D(ucXR5{UQ~>`M?V}c9rsMfUL4w^`V72g7^G$njnor z>osxO45com++BBVMY1HbLc5Z@cPl;rlwC`V(ZPlx^n)n;)~~q zoJ)ST+Yw_Au%oo~28$A9mcQf#rh+T{$-{4ZBH~VY@!bf6$8Sfn`ymiC&A$ zTiv>aNV%MsI$8-?UD%rNKf%B&Ch*i36F7pIkn;u8x8LY6*LJrk$%+!yl7O)Ai1NdO z;6DnG+J0b#$Di{nF-`G;6|G2>GcR0;UyfBjUUxnyLobnXaPryn>2szIZMjWloh^%! z;V4SIX=RoGNBNq>EC7bBW&88=!cyaZ4gnqf*47&x<~mo`1LF>Q z`6xg5IFpJhN)?p@9db>6QaXXIj(*0%0KQOX7x=05;zcO@$fL@?mi<3)ilP&Rhx7)u zO@j2plX7n?`BTw{1Dl@zj03cG7EmAj_-E|*H{EPU9C^47Bt1{2{Qc}1-6-?nPk!3& zy?2rwcG#hIsm?FKw4d!aNB`%EvWA#yJM6HdUGcRq*={@UtTRKF*?AX!#eR3w?`_10 z9qm^?yT;Egx$AD-u5!hWtjlq_DPcB*S;C9?neMnlpJqA< zH9PK@BXzsbQ9el@pltx0&GNCcKV!{OqWtjcD^>2@eS1*cdZX4N*}~#ukfTgbI5THkzc7T0a$nX*hrU1@&|d(HW!!iG<{`Ml<(IyDcvO@9YaW$2o4?D-Dt|q;wAg(ksGi&a3uQk^>XJ6;+y-%*Yf83Y` zhmJ=C%T=^|!92@@Dq*0zBenJ9FWNC^&?D)5udf_N_N+ffZ)z*?3|qoW5x$_0R|1F8 z#d9`Kl!np-@@El#VXP3*Yo_eeaP9Dxi>j-J`tR}B9O2mKuP8~vE0qX7ga4IBUFD9Q z-*m?5EJBYljXF}^3jn)ABi>x6b8`0w5|DJT_@LuD9jMj5Q@(F8TZ83KiN=Mq?g& zwSQKtHpUDwhd$!qADrGcJ)$-v_1yKEDI01)?n$J5???9hI1J5isuUBOzzol|G*|Ax;x|H=8c8V`;#bseEm>X%AB38@Dd z^DC)M2(c~on%kSyxrn?3^GtH?f|ZzvZ?X|(U2m60_@$z{sW$rbn5tUY8w(W{!WO0g zENKLcTKwCrEW35Ekp~6VNT8A+XBm?p4Eu*0t~VSrhrl>DoSg;R7uR1VIoJdDG&(ge z$wQWNnVjK+MXyy&jGG(|qTxCU7tj$g+&}I*t&fnZ8pDh=15SAN*=+jH%@&G)ikn7H z6Xm+YJ1?kn#Y#yuFD~Qcl(k@wi96u}K4&lf1Qwwy+e1W03d2oZ=^$FZ+?>?>?$S=I zO4!IR6YASSs`u)e##`E~tpY^m*I770IFoL{$-^&$gBY_j-%V`Cr4$3?fV26v^?XNx z&R_q(7eL(BYv-ZiFLvWQ@faOWp{YvR;l0BQ$tU@~HE)Dlsk(3NdaZ|rP@;gQEgfgs z24Z(wy|G#U8JvJA%(8lc`E5uIDSp)e2Km%@tu*)H!z1`rCf5jHtBQto`!*7PG_Mti z{*^Q(vX4u0*6?O9elix>R4vX;b&n69JV*SVK5=kzYFYQ$T)12Hgv2v}6J@F!m6!X! zfdz6h9~xvL&WEiWlw|C`RbeIo*7<`}W& zR{6SW^uq0Nn2CU=gOX$OsUYVup|!m)4AatBJAi41qNLC99G>6O5jQ8EAJYB` zg=*Z(a!!*Ynd(&-1KxcwYkYqP6*%VE=?@2&ciuLMcrl-Kf7+{I^Da5>K9brCx%-uc zEWXyy{j8o!vFRK7F$`-!d2)r=OZ6->ziM9oEc*mk*MQY3Oc+SflnNp_n& zG)W*^O+lZkv+Re~#pIS)yO&6+;qdsSWAX3Rr>CCC;h-PaCSQryhd!rUJ6ISe@XfDa z`)JLFG_)H$^k5Dzi*5ci3p6d43vVW(wwXa1{qZOzWFjv-nAs`hT70yK17$b%-cg0K zf1zREJI=WOpp#1C312OtSs{yb3Yd?vMNMJ6Pn`PwWT1-7j5kX@h~%S#2OJO3il{hc1jXO2*sCd>Q?mXdTd}jTGQZ)N-sd}Xsk9W^5WJivU zuyOd|yVD}2T$0+SvPHzn8}-_@y)kT?amF@puJZ(a@MMDaWquFKRRX%DA98#vuu>*S zer!X^FUV?s8yu$EPZN;tkVb3Q0Yoy9mcgvZoDk`CY`gno+X-!qi7N|s-8VaU(-?_* zp5G{8hR~=6aByf4L^WLZp=k4XVT}P+D8QQw8q}e#dyVc9(ShGima&B)8X8WJNopH+ z<+BaEV12i=NTp177*M0nKb?NHN6oAVV0wSAwL!@xlQ^7uD~wEg;p(}UlK^Fs3MO)k zC`qT3&(${Yd%F4o0NT0{aQZ4f>Crk`1Seth3_1dsubI$U6;d3oiL;I?5^T;m(z{1e zRTd6wU6JQzNS!^J+%8^FvE&P%^p*|Sw$_f;h|yIe$P}3<2@wI==&;&mO}AQx+;8fQ zoyQMl*5>Ixt?3@41QRtdB0c0Wz3TAy-NA^ITw5>^>l5o|Z^3IZwQ+)k25oGKzHl$X zI0ridu3BCh+zZW0yZ0(R=nT@L?YMCE?^w?<%xCUtGuYjPPLNCeT8CX12EfCKG>ZHs zFw=#n)r*he2l-O7++BB3%LiY}S(MO{)7K=66vL!e&8tk!E_nH)eWsTlXMJ)7fAlEW z5nr)MW(r>ZjUj()zN!O}HbYI3rL}HrLzC_A&%;s<_%^up=jlZ~WXLndO!^(f?)Vne7;-S^(%CpC)eM*YVW&d7vsD933YtJ#!tp zJcXG0ji_N-Wx8K4wCSSc_+HoWA1BWu@v68@?u*?;T)){}+AdrS zH2cN{yuWO|D+9vyafbP$E>$`sgJeFR%1H^{esW%0S$#>LRSfC6dcFBC%p$$ktbjKn za#bMc<_o=;M|cjrn`TUS@7LYNoK~~I@nTkl)bBdECxhoqM&>s&#te;i(+$(+ZprHf z*K(4F6zXBmj3-X(hXDb|fKA(B2EUB1o7xM3Y9Fl*ltKQ6(%oTE;;#$y@{2EC$|Egn z#I4=PGvo@beaH9|9E+Nx^Q>%X-O@hrqDX4~4(;j88#oh^oPv|0qtFaJ*hD^^eI9pi zU0Lc5OwK{Hr2csareC+dPIUKK$=aweb^W}&h=Bc~&jxCp1b${3cNl4wSsJTgwWP8rwzuTeR=k*!%UgU!4Fj%ArIZh#b^q~JeH7N-L%CDJJx&|Ilx zUiiiNZrK_#Kyki$Bz=__8_OdZ+9>lS>UcQvWE_0B!#8*g?x)-u0J^{_XouG_gT^S; zFg}J}x94l#5EEOMQ`Tx}$7YWOOkk+X^^eeUrLxtI4!Rb$b{@vD&NRHgKVDQ z=VF_)ysi3*6ME+c$I4PA`?>I1?k57yG5D37YwSEQo-()98ga6$MjEA4vGg^7YI|yNjup*(ldvTSL-Jy3?_3t4(NUR@sN*BiJxYMB(9Y;^nW!40Q=mM zd~)%5z?=4js}!LF(iV@rO3pj`@g%*IsBg7i9k1qvNy^?sYRnLSkoou4`Ajk7IZ7qY zh26*(@NR7gb6ZF1>&S7S zF=ZMvzUwwOS5kMs^~qZsQC7^?%Q3x7G5JZJ8pMTPt7!(bZB)76ZG9rNiJeh|lzXs*YzkS$JH}KE|cF)T!Rl{su>Tk>nO<|1MdK2FOw(Rg#W_K-b-G-s!?X@mQ?iIeFu_Khr-Uwdz07g{lPXG=G!+KXCdwGD*dl5N}D@P61?txK-RLU zuJZZ~C4ZdOrvh2VHJH6y+zIGqjPpn-(7wZFgqT7R+mzR6=Nb3<=S)h8dcIyb#6 zB4Y1nv8Mj8WjXZI^Wey216yj&<|9-qQV^ftCd8MEWvZJqhpQPB*`gVAZxx_nNF^~E zW3yKy#h`t)&a8Ti_ldAx8DfPCuSHIFr5S9)F!TJ|~~ju}*qYTY->ph8w~*dtgg@ znldW^xEXVCB~!b->u_caC+?$gz8`&)7N`SpD@`+j;pN^NAXj|>ao~9w>z@5@F)n`o z>ZDe=WZ%;_k%iVRre8BDx3(F|lbK$R`$D=dFBg|k7ts1wtqoI+TOg;;Gw{fmWCoRL zRP9{HaF=>Fgb{K`kButK+nqS_9VZH9`*&kZ+tbi7_G3vl&f!^c?l`QBe$}d=<(6@BF=J0n&5@Dw6cmt_S1ZAH)sf>rJesXqHCse|#c)=;RS)P#ny^~@ zDG26n*}&a>&IelD5?pdQ0X8U;sb|i`w#L@npexGtp||i^QuCW{ADMAM*5s8-1D8O? z*S0Q_qwROCM|)Z}KMS>9KBVjMs1Z%C7}4Rf|o|Rk+QiPc|ioUa3a1QeVmEzDt!A>lb6A#-t;QP~P;{ zfp0AOYa17y&lNh$Y{dsDqkHp*K=3w(N;H_WnJgcVe;Wb+sh#sS)!767vPFn3#kDw@ z;y;Xifb1%4_jk`*JjNJ&9;o{lSaMbSNl}VSZl>=Q_7)+(M-Yy^UDKmgB7oAjj9-!o zJ(hN=Zs!x)9tknFcH@0N5k+{TQUYR>#?0k1#0Q`+`62Y`{_}IPkKY=oO6<2t@t}E) z;v=c1@!T=O0koJJ=%bAZx6^IsFq#dm6Vi3?@lc+f@oins-YOG#_^XS|K$>V~kd0qQMRvMj&JgAm9##ORV&Gm+K3w*mWypkAwEmW9-JP|+=#HPrm1rWIM&HL6q zqyWlgtoQW;wBCfm7qcj*!(9D-*oYdYBqIC%78w*8-?Js_-KwPq9&g>4YS9|+a6tJ> zme>kaqb*#oF7z2Vdh-!~Y?udwIfsl=u;Pl;fVV8ox=Vq!4*!<8xC>w7DCFU>Nu2-r z+=`k)E2j2$qsDz>FUuT)9^4I!Os=7AJ1d#mF(WsCUq~*q)b-_{qJA@_Jb_T>Gbz#2 z0<9x|%N(6q)6_dUze^RdJ+2lj3?gJSW$z7|?Idp=51^ao)ibk@0?tB4(OYVUq~=GQ zRE9u+Z9dmrB-0HpFE+rh$?oAs z7{Qu8#m-as6H~HM^c1@`oZ;rG6D|m(etAjN7oZo?ePJ`8B(5%1kWE| zoB>YA;8YTG!C&Ch%7>Z9x{i%xvho5Nf`>ouc$Lv8 z4rls_{>+Bke!z{G5W(qOMptvD4EX#5?0X?U0sKKXoftGIUrV>~ahJ)KGg!JB{ID%{ zUtY9#sltKeK^UI7jvs-e%qa(a;_@6M{=jJBfpOR=_$kOw?m=hELZ2tBa2~5Aj1h)c zWhvc$zbz?^(=S^O5`f>KdqS(3{b;eVGc9}ZXvmj?ukyQ>p4pK<&~o@@Bym7!?W@Tt zr~6i3*U6@lF>Qrt1yQ?%3;)JEI2^kfm7bbom-oMC)C(13*||AImy%9938C;}zl@Gf z^`Iq#kSCDGUwjKvcL$w;@#LzjG(*}T_MO-Im(<;iZHPtsvB5xhg&Bcky;<@Qyy=b< zF3omwO9p2HPR#18W;xV zu##R$)&9j_mL*H|{|1oB>so4j-rE-~0{VJZDFV8;=G_oICdBAwGzzrc-VMvRA3np> z_-jQUcypIadvp#mqtPtJ$VL(`pCz^uu>5-J9sX2vccDiTBvUu>Ak(>l>3r0hKrD&{ zda)FtkRTax;_5xs{pMrQ{V6uVsr&o>5oatj^pzh9G;*W(AKyQMZIqA3Z}bqtNEt2q z|3q(AE&us{wD4ND1YWIe+UhX|jdb0m#d=??_eurWl=BLoJX)|Ib&a_jTmMbA!g9fB zm5nfqtVxOI?EcNRkw9Du)bQE;v6kF_rj6nU_M0`|4y6t8vq}7UtUN~QBhS^~-8MQq zgB5V?={MWsa?s$WVgVdm!IvKoT?l9dV#P!y1BJO0pZsu3<6iG>N5>kgU)~6Siq$CS zE=u^{b4ALO?-;o>9!}Uicx+nO^-Qk5J}d1)kX2@*mzf|_uiu4#!z71$Xyfr8yv@83 zKB)D2@VC?`7K%Oj>%T=#r)AnV2TY!OEG*gt*SlM&tc+(Ai)H8w>X z;&wvh3mpJ=Q)?4=+7~wBGhA)?NJ0$B6jM&`K}(rPB_&@I?Wly>P(7ZgP&#s~TenAZ zP#Rff?n()2Oc%AO9)lrqUN%)Mvs>w0pji0kkBkr8#|7N0pCNG2ahK;y_2L4Rrr!xs zbHKfo!xTzvyd{4C^^Mj;XX>#-nhU0>Mj4LpP);xN#yFl|SlXvu*lf@jcknee_Lf?a zKAVUb!lfAH5ce^yrYB#68~vsE2dS~m$#|rfjEkx7-e}pUpEa0_KTm4fL2btPcH9DU zJt=Fx)X_6;Nmd_uP_ShUK$`t$BNlFtvoYa1=S{|Nvh2EL0y4WhqK z1KeL3-y$oJ{UIQPcMkG|K>e6;#&lO=Ref7Pn#}xU#s6O$+7`_)B?^mYn*I8H!L=Q( z7`Uw#Er^Fn9o|(2GUqJ%zVT!FT1;m4aVMMaWW7K}<~&rZaZw|pY8UJ4PuVPd#KPC(yKZp6ZjOZ+QNY*O2o4i-V3A@>mNw+{>?3a3akMz$m!yV! zTlqtk<{lj1QeX?x>k#-fu;!&4edNDo?)mph1T^ycRn27{$zurDb>1ic)T06LVL?8+ zbb6>gAE1y?+Bp~(FL;l;8UG<3T7j$ zg6%lvlW7{=LAk6X4Uf4$&njpw$N3R-%A6KqrsI(OEo&JIO>@RLSwgO#nl*{1p}6OB zNC^SN++Egpum1R!-KzOXQ6q4{7mM`EvI^v6ErK>zd>&R*fkXjXi3h`GE`CUCr^xqp zK3-yL+dpS@&*?)kRjIe3s?;-p$Ju6jA)u^eV%0QU| zSXVysgH9zB3orYFw0q3nem29m<2+Fa@R&ML)+0ru=Mp1z$9E@q-0JER|IK@j;;Yt* zEzjm#yQ~>Ak}9Rlu^w97hbna6K7@a6M5sXe3a$3z5yQq=0I`M^I6UHv<=JE8pyf66N}Ly=RHG1Aeb z6FLSog!l%G5D5lxwEbEdFXbWE6`K={EXsNSvW-d#HBdQ1$cYDBe zgzYnUR1UwCnxK-?Zv;fk2k-;5xzfkK<>eH@o2hBK%qS^E0nuCZk%U*q=^`L}rD#Xq z6h06gDDtOpde|3Q_xHDejwqUJ0lI8t0WN+GPYJow`qmP-YnOt zVXi8WCeAd;u63Sif9xe)kaBd!eha(f)7t_-h(L~+b*22rE=^`{SnXiBZE5?C;L=aj z5f~AAI;VEOsD=S(?To<^s2&T1voT^FeU4+^ASv2$k`Oh#2VK9@p7BgszI|=4GuDSpP}k zMju{#3{HcsTX>t*q>(Pq`F5H_=0%v_k z*%s-Hk=l9?!5?5X; z$=MV!(^+NB&myHVycq>Xj%|GYhuNO1EnoFhr*-4>#J!la%Pr@~JILat z3fE*`FRJ3bovk@xUXzk6D^~lj3P?rvn?1z@L?>NaDEbZ_solXA+pmilzrN1z;O7N{0IA4PfJ9d9^!hx$6V`;$bwmBav zjd#jh6Y;;-@($6a6A>QZibF$~PYBB2)|+66x;+ZnmOUmORz7!M^nC3|i;((a3fi5k zj|}5ZgB84gcV#G@R5_GKB#yK{-5`gOFeyJgfBjdp1_$M8*Wxu-m9%7rgIrax_%O`) z+06`dn!WNEEK2Ebv4G@xnA1l%6ovxuxR^Y1=gkUx?==9JUjFT*JSjZyUs^KSL-uly zC~w#X5ZJE^@J#Uk{ZKu#>Z#nF^_e+rW8*`Er)f{Wx+JDSug}*>f?)hU=RmIqm90yN z-!=4Nny>?!q{kE?;#NM{v|41}u9@oJgJtrRX4|g<9Af@;s8U<4c#PkV2FVzDdNRGf zr$ROKBedPsp+**uZ$daE0MYk2;5tAulkHm^oOX~^)7UiNp;#;*;P^?WV@FYXT;qNz z*{}kFoi04w-}skt1|=8YVYof41MLo8HFYGFwp{p_OnDV9kK)Qag#5FgE+%ocG_8Ct zSq615govu?M1&TjR;bMOK(l9ma`g6Zy>ui8*`=!|x2nUR8@}BN45XqyvTdj1yXhk5NoxC_(;+^8DS*VwN0c(`~7~vjJR! z<7R$J@?p!O3HT8H{j-AXPq;Jaq3BQ4^jL-=qwD>s_1pdVRwNNGk_Xi|s=PD2?FJzR z?zKlZonR{|Ll$phtmVLwD522yKa|A1gKZKBi%Y4;S=Nb6tk>r?c*u@6jy6$sL{$HT zU@ZP;5&*1tP|^Ah{=xTsa!$d>0b=$=`>1;FWr)uu%6TJ!;>HAx=Y`rePrQq<`JrO1 z7us}eDju4*fuJvbNJ$*$pXAFCN;yphAOJ%#8T#_Rn2tJ9~=DUA-`S6 zW~(Ju3T!B-JIQhZ=99LsuTltj z_0Aqr71M7far0Q20!)J)>0GI#%I9_a_45Z=hw66h9C429%5+d>c4X_2+qHc^jKU(Q z!Jzc|?pZ4KQNG#;{!_l#D1+i#S)MfX@WkJ@Wxa?p^g#V38Ci`W=g;q_pB;px9LFev zf}LWi!G8`zFr`1AJ$x)1df#dl>s z%_!3#U`6TM|Au#2T*%J)_E{6SUPFUc9;bxWP`l$%474jR>yGZ~znY4y+|%L!|C$2w zK6Lwyf%o4}06%mUa9yujKcnc)3WKSP=1l|st6y$+FG_YIML^~O%ORf$QOff*k=mPu z%pHHwLnQ(g@I0}|_c)FJJX%?cxaRCuCGo<8gr_em$t6Qvu5USXVY=qj#C+Hm}?Gxr(DmCTnMpj1-L;^X3c>jzq-XisTX zI|%LYXyWn!Xgv=z+j8T>{!+K&^r{+K{i`ncFN`3lgaVydS4pP1xc+nmvbWIKLo&YN%DIE+i#SPhb&aqKpN^cV;Yk*_dz@4Ewgq7U{Kw1uZ6ZN#p5oz6p5 zKkrbwCKqE7kI3ys_8_--v9jh`b!6EWR4&$=*$jM0ynRulBvB&sFfWR-&6*SUtqf`T zms?aUT_DC|4o@+sx*~Ci_z8;@H(x;Rn<9-(YEXL(|GV8(UfR!1B3o||2Q)^ax6Nzk z?ZYmxev2=RM+Xe50?r7D0`CcyGo{S6`uEJnsFdQNfuzKoPAR9Y2nJl z);Wjf7Fna|bXp`iNY#98;hz9Y>6C7DU%aVZ1EdJb)}T(T=(a4#i1HvB*VRiq+fbnj z7e0w8Vu^N`9vsf1pKY$^hH)2J=h!g5P#8#+U0l1sU7!kM}+ z`2=Zy>-SS>36t%|cUw8P3*LWt=1H{EtwGZCKecnN_5B4X4)@AHIuo+33XwlU*QR-s zXE7?NrvP$}bE_;T5s}jiv>ZOh0Uz{Oo)eg}n}?zmsBvpY`4W{@rC3z_eo%YU@#cM# zVvUDRhs$Vk?I8Rps0`hub77&IRhpmlhW)X8#GJ-+#h)!UWhgL%7iA5{-#PF9xS`ub z6QQJb!Itssu3O2;&(ddVJRHaCWHPh_@x~5mqf=<6*by|27yFJrs*6sma&=#x+t9Ge zVF5!QM6R9M&CV2qHcde`^DQP2+OkV~Hy^@$uok9z-8YiAr4j8)^`(Aqa2hGO6kT7q z5(K+GIs49poG+cuaC!Rsv-LgrOl-C9=kjF6^<7&9TuFl0#%BT9kz)vFB7S#O%gMAg z9BY9{9)Es%d(m}@&K(sj8fvgxYv^1ZM9~PVKRRH_*3o6yM-8iC8R)KHBZ%sM!bfh9 z$=o7+-dpIacaSR7L4lV|ov)h%O z`E6acxX&7wROq=sB8>kKYw1T9UMaJi+--EhT$fh=^(Kz$%NC~$H5ElYAlajk!_>WQ z`Cp$6e(q#@o~{9_8Tv;(#C+*HT&Ss|4(`KZ79Q%(=yy~G4`bM;sdy zQI#gOYBeHrfpEc8+~*e;DugzB!03C!H7&vRT)%;MUugk zh%nfHlOxlV&y^l-buExve7Z}PY`(Gi0ts!+BRmVe*?}^3tC-FMuARLx1AaV=MH5B3 zKgLt66YqaiPV`N9*}KLgZwP96pJ%@(65q8tnUv-ODIPOBYkhQ7Gpz?HE;hq~OkZ3{ z3i~tJ3DRF`VaI%h;sgDbT97ExUA8>eANgdnaHW&9?iy_jwcx4r?83tC!Hc|&KBk%xYYSD27V)kZaE5~AcS12b=C+vfaWm#TEhW($Z4MEU@Jb?=%Y z2Uz?s40S6KZ3ZEJue2yW#8Q3s{J=zCCIV_uiY8!$H#wb5CVwDlDk6?Wd6sQZ$IT+y ze^fkZ_2TET8eC5|n}Yv|K9~YP?R;e&W%FPE{TaP#BnV8nu#cRyD8?p{nWix@;i+ey zc`aYL$fA&R{dZ=Ezn_cvN<#^+b_PQ`xliu7>8Nq|tGZo%nk<16ZfFi2ac@G8_8xz$yh>^9 z)Fu?jiIc{4)sxL?8ezDKDI6$06f|7-pR4J-?-JV-QAjb`Q0|-uQNVoS%z+8(Cj+DG zva=0e6cx*9UQxbmnL_=zfsDO+&EH%ye_;+l zCFe5dib4qyL?cy|&eP9>iQJDU3zYTtH|`Pey{F9<_}xms-E_d>6W6zxX$i!;+it%_ zpa?!d9-pt`OsTyi3Ft=_9s@x6MQFC|IT@Apn znj-6ix3%QLmGQleBB{bLVzE=)P_a~SGvlqUiwCWvg3B<*J-R_h*>H8|!+en-5qDV^)OJ&okp#NW{wgDSWN z)2^Y(C3zlZ@wIeIQn_vD%Ce?J5g5OjU4q)RZ8CYhd0M=;a25+G}zsYz6CWn1~1~1l}6D}IRBO< zzu6g#eWh!c7yBln(56}#@BLv6`K>RMd-ao=)sEi`N&HI$iXxIa9?l?clG+M3NyEIL zkToZvrE$<3`go>Aif=~qfZ5=HWDeSfS}1|n@3pdaWueB6b}P}YGWz&u)W43e)SZ{! zf4-x~ko@#q=lGj>Q?Y~SPqqD0=yV0bN?`#}iLe+2GJUMLR3UGq179=O3mq_>Jsu7W zFRz_vJ4kFy&~i+a(^mOE3{tD0T)^~Kk5AKam`f`kNZ8sD(A!9uHAoDZ3NvVpp!wL2 zC>6Rbe$cYg+4(Z?D@Og=w>qGVK5e(-tvBA7yk#yg2vTZkRBRi7qkeZ)I7nZrhxLX! zOq)O?TygFro=1gf;$EbrOsvYAdp#}cU~4}I75Z(5OK;PH_wJE6=-MJ^x&7CYb;Ry?;Tp7@8lp|&ed?gOGUQT83GOzuD-~5-0ZlYwyOT(Wj;4y zEOJ)dn&b5L?d07VZ?{uG+@FPSPNqghkPQCcF$Sj1-4)0P(hpu$dnDIHqps)wvMi8^ z4_a~j3-?8j>brLD0IoW%3-?)m3JT!MTcr5pOVh`(!aYr62n>8zgV__q>PS;p7tNs9 zi)P*96_cBbVVzKBKa@>10E;CX$o9nWoHEv&cC#M{1_|=Y4HBk!niwI@pDF@2I&9^7 z^ldWj{O0D$+bb(8e|@Z|v?>ZWNjjDxoDi-;wyrr5I6$yI)I?n`kPY(&NEU(bT*MJ< z@u|$&Q|U{W0&;35A6W^>bn(;%E&7!$2Ey`5DZa&U+i=FJI6yQ|Yu*Z_wI)UfS97)_tGO_omM` zOQ+F&qY`1mDu&KjoXz@Rm;()M=>-NbEzdmGQRN z@3{iEaQ#>}ejezjyjx&EGRjI$^jow?+1ui~fBM#^gcCmO^sG8LW7M-T+zD?}2VD*i z3-g1Y%td<#V+I{ZbN;H9Gk#th;CV7h7#kOid@Q_xp+zB5k%3qIwuqRe=0m#~gJT4k z>B~cPe2(bb50zhD|M3qKI69>MOcJyelGyE44>5BHiYf!adDwz@LZ;uf7vMYZUrN5{ zEErHF)qp1&bq;9+=lCYbiYkk}mLG8{_beJ@OVI z;}DGx=^tYF#Y&t0LNlxd`;hs!INjKD^F0BWzBl{&6Q*X}A)M%mDy}VQ`B*BhbpqBL z!ie-s;H=6JQtyL)SZY6JMlYv+0Ede9^6^j;!$j=iRl`;U>SFkz+D~GISxA4`4gJ%* zBN`?(&fRwwcwMefUzpimUtf~F)(@p%q`Pd;m+G_mhvqcPclVTBOCD_b;{KHAKu%Vj+{Py9mxBvGp5w{#!deWJ)I}?wrAFb#i%8kh_LOU^P+@~ zc$v~oyXfnmWO;H>BJ42Ug2c?F?Y*Kn=-IOLCtO^^)t4G43zu9$A!@KfX+^SOO6PDw z>#hXwkWgk>{3YWEt`E-(?4SE+ykqj0Ob_31?jMh%_T|6Yx)c@j*7Qek;$gG3F&de$ z%cjjg&G^u9shW}{qUZPvexJHLM_`v|mE9sl>2K0D{rjaa{Vivlg+D7~68IL6&uiOA z>DwU)kD9`35w+1-u4DV#heYB><-95n8^_R=(Y*w^6|Z4r zQ2QIL9o&!9ngUcyc`NtRL*PF3Z+(wpOnEW+%^RqP6e>sWzvK&X_-eWKvb8S9{baJr zvWR`wL-!yKwbcojg%mw;AF2!?5*=0^d$8-nJSR)_f58Hb7yW8v1GvB;dTlMozvZC$J;?H?E! z*W18I7JOlp$F@-oJev@M?%nP?p^5i3U9#N(qM)lsCOk;qc6l6eXmWI|RL_!;Y=GbR z_+d&;^~2+^++d!HtM3WXC&>Jd!CXD2#}(bc)@NrUE-H-TtYZ=L_bFREm$)SiTx5wX zGM4#9y)UrhQ!XjW@R3bAx*&Rve4Mgzv(2IG`cGhz0xCAW=#Lmwnce8PkW%GYV2Kul-L(4gpMt@EpZU2xZP<~1#|A!Kw6)Eg0o zavldE(!@qc$Y-M%tC@DjS>O2GeC5>=#&<~(QzesszMSyE%)h)#Ummo8F_2NCh{mAT zK)C79={pZ2(v~4)&cfG&0 zNbdk4d`-~0LD=BejL)Kr!5Bi*SM=cvAj6VLNYghatyfQ|4+vMyiqVq=6fq=|t@EV1 zI`NSgkJ)~QqIb;!56Kl=HAWNKZcDQkYF@Z>mJ=z|i0*^TE|^?83wCGF%Dg-I7qw2Q zZFdOXA0|0O+HBEmE@PRK5Kbll06LKs$ke@DA0aT9pbLbPO*?15(zMc8jkf#lQ}hC4 z44^k4#CXhaU6e-@?LI(eaanZ763Bg>M~Ik(B4cB~POHfrm%M+VhMP91jWkIMVRob3 zWnPU%Br+qV8kuyWq?pdOH<2OF(&K^5EW5xi@$q|Caip81&Gq@LIoNGlC%j1FoaBBW zH!Pm~3o~c;5Giy9yzZ|MpJHqq#HXT=4u6(tXzqA)n>4a3?}3M7l%YS&uB$@5B%0^X-aUFrg6g7JWtmIbvt7w%5?iy3D3(5 z0|N=ZU&(z@IO68dd2P?=atWxDU&dCuJBXTebpK%~FzpFNP8qhs4kJ42m8(dx6iTio=-=Sph@Sx;uCXK+7i)=xfFhp@ZgKHU2M`faC$~Wk1NKax`WltzMB_HCDm(z|G#Ji z^HV2ge6kt2ecWhWlliyq>g~pGS?u!uY?+O?zFegA17BYf@2neu9g}QQ!zfQnUel8n zZo4b63#3J26S=`l9t=HOp9V!m<+Jk~62`MWys{8@c$CNjf2)2dt9Z(zF`$%Ff9MQh zTDunLWO=KW*XDNfb;!*9#!dwn#U{;OgKwRh5x(%^qhGPW(mqk4WBx$PEA zGRI5`nGfmJcrNhvIM`R9=IV3toS&Lsch&pgB$}FZBkwL!!VP14#sI$n!GsOP4>f0t zcxM;lP^z2PTPGRH8D1>ZIvgWWqtbq#lS9eP^`*-?#6>}UT&(Mfs*H+c5BHX*^9GN!N%#1yMxc@ORs5Nu?>;M~Zc=j) zOff1+lK=XCAszh^U5l*DF_0@rS$UZF`*g$~-#Y&JuD8)L>Oib+Q1(y8;pOk1x$ zsv^@E4jqW*mTclvv+{w-bDo8DukbChV^hc_Hh&o*xQAKxt(d6UHUzkgn5+7Fw#w_po z&DV2ofv+UqKj&po^_%*D#J@)2D#Rn|pYFCoJ0cr`3{k)k>675>qQ(cG%3!-&Npa?6pCC0G$sPI#ZjA;4tb1GC<|F#8ZbvOSJnzHz%nMhYQo z_+7*y%R{nFf&T%(z#M2!q#=i=Kl6OwFC&E?N8s$Xtz+42HG|{Lhn>A6G9f9(&H`n; z#Epr9e5_m4=H@8(`5>f6B8JC^fnz%k(41|9iRRO}LfeuoXvAg;OFQLBt8E-CZKY)9wVti1vaq851r zGQNQ1M(^={1A1pr$>N8OysTAIcvV5Mk!2t!wo0PPz~Oydy&9ZMcI_2x=Z?GP0^xnC zm?Tex3oI zYax*@!m?&TmJ_#9rXOw>+ZcVNL^UNNebbJNArJQMTLNcy_7|Q5#`-6TD8!X>nKuJg zMrIU9np)|<6MIT55kk~Mdn313@6Z!BdX%HH=(kqMB&@9Ad0n?(Pk>^^*Cva|g#zTL z`!2%A@wbI{Sl%2UdYA3ZAP*1nlAS~cF&SclBX-6k zUrNO_Njn0aT;>$v+^8fX$c;69$xk9mUs*v)Vz*sG6!_>MDpT-od458bH_?uEphlft zXRXnM*=y?hC*xxW<8wL=)rm6&){gRwpD9wlDcD>#nWDc3oNGSGpdmFT{k3C5P$|_| z*q-r1s0kzk zfluL*`=d(wruPFhPg$WUGlRdj4&NHK>V&dMH22NW7ixTtEcja+Br7Y;vk!=gV%5D~ zhY3+Bc$_C2c-`0ue}XWBQ2Zb5l)tT%xYIejY*ZZQ|NA~EVoNn_rA(o_RcDtgc+tsZ zP5foGxg)m&O^a7B)`&~c(*1OE7MToBE*YaC>cpJ2owVih+gJz{}TEz zw2B_dVB1U(dsg#i$@Ijaa&^ME(TguZk)M3Wlk7kAjC_O(^ufXwnT(P4aHQb@m==yh z7k`~bz!NbAq3Vjar$_*704k2rC_q4@YHc~ zy{WuJKaZ&hkuIUX5*O-@ZQ8$V$KU4bkmE=16fzj3GrSq!EB)>U2cGN%(RxH|MGd>| z(|v+m_-{EBVL7U9{TT|7$q-lw>YBxVw9<$qgjKHJtKqdh@0RXK z3{c*XE&=yzML@2k0ZT$azT|`y3-e zQ``mjP1uC?V~QlA*aPK2|MX7P2e4|()cj>=zLUu4AE0ewi`D1|drV;AM*h%gXb}4m zStQYh9jTJY5X6UP`?M)P4AvDf;BXY}aG1;MMMZ7Mvu{GM^z`YF&U`8H$0dvE>jITt z1I^_185}x}YJ;WQo{r=(SJIv(<%C4lxQnub|J{gcD{fU5k;cf6LncG_7RpHX`W~+L zhduXn_c98|F<9OS&tIWO64=0F7pljxt&3IzzOVOMdnpkapYqbCNbw_IcxAPI6LRd{ zVhAbezjHA|Q9FM8NH5Xn(M7q(W;J{g zAV;C6M+JW#+;#=Zx zEaYjc+2eB1IdZuy_l?kLOb0(h?^mN+PD1(rN7Gp_MA>%Hniv|SyBq0l7(yv&L{hrD z1q6ofZloooySp1nY3U9HsR4$WGw*lK`3DovbKiTfb?vp~#5lVthDbku2UTeCw~>t* zi=4*J8qf=J6amQV&{$Bn6fmyG`T#D*|B|9dP^jJ;Gw=P00kd+8_34)zR{`^(*ov+S zr~YQtz3UBJf(>kJ=G3X2osi<}(xf1?01Brv&ch{h;6_E`!M$$3^M!7W`R5Hb`0Yci zbp8Q1xLO^8!BfMA{K}(DyV)=Jl=O^uZ)6DJ3B&C&#fGHmtW+6nDkMRFJxGArIJn=o zZMyWWd@}MTzEP{j-kWId}Ws;;wexlOs1|HD@+2PGb!70Z8MMEn}0)Yvxs%4#kt^8(qwa=$)ryhs@d)#+&8u;Jo{k~2T zVp^yaOXTFn&Xi0FC4C^gZKiXn0q0Fzgp3Nw=lHsb@^cYRYB&u6y){_=e-^-#u(}C^ zPTi8^ZH_1}13R|V`bt&bt0e+brEYX0Jz0D1%gNyl(y)j{@@~82%jF#_!j-cb2LeW7WC zX#cWgA!)~$4y0?@jh2_e@=2ql?fhR4`!=0l+Z@+UF?6h)^z}Z4v5*S?uL*tt#$@*B zG&g}swguo1fGL0T#2jxFWaU|a681L0`j$tJnvT}=B5iO2hV}>4fqGja0)NO%gFe5| zv6^o6#fZ6dtLJXM5%q_E*?-Cs2k<(3?zlhpwjaPuF}kPdh>U%DSCdRShOeaZjC0by zO;^h=(Cg~cayEurxRtK;p7V1aX}xMO$%JBLD7Mgz*QYSn>1c0s-j*Va2rV+aeg)X| zG+C~sbC;0o#{7y&YXo)5#&kbtH1bsg8Np>LVY8?Za&l=zdzU276EK9n$Fs%RQ9{kyduaXA&) z%|}XRyVex#2E5g8pV#Uu0@N@WyDQ;8zq=YpdRDh4pdsK3qq;D7-CHVuWADekA&yGH z%%Q0ndRq!HRFki3wQC7eLfrNxdLMLGm^g@Mj#!x3yz@LuuP+Tj@Cx-9|I%}hDy*_A z?$p8}s1o~21aQP0?omZ1?JlGCO(QDEmT_F;U9ZE&My+0x!u`d*58EyVK^ZeSERk54 zM0)RH?g&U8U}3gC42+n`v*z_%tNVK83WQ#V)%N-rT`Ze>N*pX7I3Ld#L}RW3zOu3k zW)^^&pr*nvJL|dG3PX?l&|wVAn-x>I#BtZp`44u^4;4!y-m`Rq;R4(G>U-*1nct?q z6!2dHeWiGXRBR$ILIQifkBi>(rObss&;EC!XMTa2%4+T-zYAlzVJU!^^*j|}UW`Dj zGS}G5xp5?|#|I8~}g zMU%b-2t$l2a%Uqbyo;S_asV{Ik#KrzK2t*F$zG?e~(6jfqG>Z#NNn=nXlw1$khan(SO10IVaS_!0!b1SfI-skVNL6>bug<2Q$J}M_S}3H zyqo^i^u(GAf<0g0VyM`aSRo{C_xuCS7~uVhvZi{J+jM?U$vn@Nqg(E6(*~Vf0j)oO z8N7hO^~9~aiAp~84D)X_*Vk((WjO;zu0!Fu+sk00OG@cKh?Sp{MSWXOt%8NQjDv@0 zQALq*WF|kCgy+(z8cR>@;eN-pN)1NXJL(V%%WdFCM~lV6DjIF9``4gfrGCqEwYa+o z^`dzEdO5TdZmcxoaC+;_)kc?!E`5wWQF)$KFP zJh^2tuC>)=C$F2i#QYia(9*-j$Cb72-!WT!@#}voUwXk$sp0)RQn}r^{^py!(g2MG zFO2mmJL*lyNLrI3uwceCe^YzHQeaWSH_8O9M?X+)2?IF_-1!(}wjT@yg{4178%P@>RzCD0uTnyc;& zZ3df3#^4++3={})yflaW8l2St#)Aj@U}vnxdcWsajJ|N)sNZu%R``-H+OO>8rG10H z+-Ru$&#VOOGayjl^Wzig-At!qyy0?9n{Q#0WBwn~7N~Y&4XhbDcM7VE2a%>m%D1j;Nba)w5OAt+Q~${I@cdr2Dn_AN$)(nS=aS)V&qT z|JM(W%j+RH=f6PBy?20 zm0~{08Kk?XQS1mk^jv9QBM=Jpz}g()9o7`@o{#h0*hXaNLf$3 zn{_v;_)UFn4LEzchH;}q6)>zmiu*6k)}uuZ6n#RBU_6!B4Ig)*N+8PfEku$X>eu9; zBCsXSKWMS2=pXM0{<2%>ON&m)e839-&~*LCf+{Kv%Bo2`q^>CXjnV9(8Ugdy*Xi~G zPGgtr;zk_bA4Lf`|vfjd)*dCg1T3#D#X z$ABo)UBnpFO7vfy(Rr~hh*Tz|Jdaf|?&rhfp-negH?*_VR7*^#@$NUL1D;QKgvM=S z{f1%e)oKpNtH5;M?{3@N1Tj4fi`1`R06Y9eu2$7?`y<04Ml@}cGuF2A*e zI|6Y2I!~R_G6E@Kqmp4_Yf9enMW16h18wPi$cM?Hp`oy z)x}%8O=uUxib81LvS4m#6I;e70s&9UPCnsbjDNAQ6bGBLrDYaWj^3MgxfhbvY3npg;)1BJY%QKM|d}XA%ZZ zk)jeH*xY3$^vA=3l^w5SZ1QgUty5qzC1vpA7SerL3!G|%y$z)vfg($X@U80cnQCe7 zU$}1jD<`BbP#cvdHSb-y24-?afF-HVmTOVB$X^6r)+#8Hz?ME)9>Ue>;L&g=-PV{-5`YYrBk*;~jR zgzra@%N-h$tH;_@o0v+T&ItQ<`_}xKwSP)kHGCYU4=R9)6$A`IW~>X=xXSE2Lg~0a zrZUFGretoVW19>?6P^#UOT|k`1Suey`8G`CkxF}M2530U9HN)9O}rF}KI)t61^!Ik zJ+#I6Qza0vA4XQ(wv?S}Rsw(%_1PSCXmt_FvyJI(TKgSdm*G1occ9T-@SnLmo`u8$ z8CIOqCpY*kek8_Wi2&1N@MXf4~zUY0%z$Ib8ePL<$3m$ zZ){2$xo<3x?v1u?>M79Z0wC*f9XsjG?6rnADFy%Yi{}Z<3GzRE`kMosKBD>wg9sY| zoF^X9f-L5GArvd=M5)0X7WAVfkDFc&t`g3@5%7W24?MgEj>x-dZT0pS!_ODdmfBw5>PDh-+^7#--_^GT%r`gzKjY?sE)v3pA)%F-k;3Rdl-~{nqn(Nn zj~mmBDlkgX`Zn1=Q(#6a=T)ZBSALgW_l3Z<^#oy+vE7y2jrU3;mf}|B1qDz+sDY-~ zerml%VnQ7c?RjR-qdy z*{v%g0kF)6TYzrR6Iu9n>rri`dkCHV?&k1NP$7yC1t!8hKl4N=n4?Q3bkH==@qk5L zb~WN6#C%2@<>2$^50Dfc(ap==_ze(kXl8Qh)i4hW@w>m*n8TD*-9b3k~) zT17mhk zkyV49dxbG$+bsC}rCs3vsv_NXSe?tQhDymhC!t3|67LK&yGkANVGh(ZjO3fzO;L{SXTIWm4eLXSy^$t?&rI?p}4 zXKY4=crM}ApCr&^mh7*=*ohEn6v!+N1L}{jsWFJs)8$K;zTB0QuyR&B>Wmip&WIdi zR@s0ixM*H63cgrek0lOr?7C5nl!#nvIHaeO5XX{u{7#=9mJU!F5BTpnN%e*G5JNu_d zX8OM=xZZEvQ3TKWu@DB9=uRrefFRv+j(Hpieb%~S=twA zmv6$*u_+}8SKT7xz%SUsSa1)+?tcibw4+^A!CTxNd^a`4MP!^RojJ-2V4xOa1$!6# z1tw596SB^>kj!3eQdGAEs{zol5Oh`>{8OHe8lAd2;z~(jw`Tm(SE&Gc-g|Xy&$XG+ zG4PeFJY5e41vA_lyGo*<6QgUycE#FB-6zmcBBZT!)0H@c4**Zh@^TybL82&_13pRm zAB;=LDP&M$GAHiNUNDX+W(PDysGKOiOP3|nDcw&HIT>k~hs_K>eBD_LW8(V&(45`< zz!d&4Cp$d8cdDlI8P|BcNYSE|xAUo1q==j@%IAYI`jL+dgGz8&fMOp0mj&Ht8suRe zTvAOOD1oA@n-uR6|CPvhK0SDd?13Y9MT{5>9efV@SYT_*s(~x4f{qRnbh?m4!o?>G z<+E8{)~}TLfEgb?)iDwg4q|}e7x04-k;$P#q5{vzQUKJ#_;sKSbZ*RaC}4MT3@-ig z{^5rHWu9nGhWFebk4f%ROAxx?qs+~Uhm z6M%!XC-$xSd*Dh0y0@lBGo-bhCin6y*2I0hVG7qIrF^4LW5$kWr#RjXVCFY(LXjwt z5>frhKQ5pUF^Zm43(f{Coc7M^2_B0l!e=QgUwj<9iF_`sT)&i{o{2%(S z>MuNUh5jIMG0%6qnB}Ms0aWB-`2^F#L(3?c1&?EKr*86`9Eu;mKAQg50RVs5Lnkp* zL@O0eUb4V#$*+2&;hm*$vZ}B`UR>%rY(b@xDaz&h9Kn}`@F|>csP!fv?Y4*D@#GIi2+H zOrk0tBGZ+|A}88)j0ppd2O_$PS&7x_ft!S5K@ssdFb*`Rpf$``5ySRS;NtIr*2&-CPw|F?7 zYV z(U;$r%IM!Q(%6!M@|HaYQc(0)gyz+2L72@V8W=J6fIiE z7q)GBA~SWB50hl)naijL6<60N*)pEqTE)eA`ar&TW+xD*hO^D*fjIDJ&&~k-qTAac z97$@Cv4z~=pcRR>gU6v&RD$3b3a(OV8c?kXMM$Spj2=fbZW2AK3j2C+oX>d*S&5lWCgE|%|UB-#{;3Tf*$o4 zA;dE7OL|f18-OlR@^Uz@hL3Wz1~hD{^kCk7x+jLw!~ievs}5r40N$jizm| zVej05Vr>vW%g_GBsh&;gXef&HED}II4!q@5gTH>J(l3=s-=|n&|1Q( zHS$H@n}$pbL6Ss9QlRfjIHS_h#YYZFPj#5$1bu&4<`J?>Db4aj`jOtoTcU7O?JlYP_ufd$KMKSE^&3HV3awP2xDA-WOLZG;aGGyRh&Os8ZyHynx2&gg`S!)C zBzy)r!jmFBM7=NaGsTZI!IB%TN`Ml47fG?Na-VcFlRJ|FzjIAIko;XCY?pj-X7rsw zeFt8Zou~s+(YI5}u54v{@>A|i$sMDhcdC8!w&BX%bguFxR>IgbnKE`u_`)jT0WC3D z<1N!@Y5pJlmd=)-=tqB~b0tP# zki{At#2rJfQZD|LtxeTy&`wF@Z!Du-O(Lk$(IGE+@kdx?o~i9EcNdu!@f&OStU;}a zsa6Rd7hEkKmpsIp-v@{b?t?+&kWS5KhN&2 z$!%mWfJ8@f@0rXfJI`#M2rRB+8s6{07(_~s(delK=JhStlV?Jm zbNclG{$v!Ls2_{60aZVnU#O;_#m$d9bAb{BE^N7Cz^T_|S|uBySaLzC_E@D@@Mx_~ z_}pZ-3!LGRt>fmb*UkzhJ>bC5P9M@qi4nQ63z?`V>!AT(g*0e|=5*mvFvGQMj8hE~ z(oQ)>E6{OlHIpPUMt%PiJsGTP9M7~O6ImGN_xX-7d?;iqX?vNCjp<+X(DG0?&&=E9 zp$}A1f;H_Vz8|j-+u5etl70Y){{bJ@yWz;1=Pt?gH)M?swRboL#B;EkBkix>*0R%sX@Qlmze|YxkdW zqgn*TXa78R%rV`-Uty+ys{NtN`L`f0s{E1%mdA>ZSLG~6k~4!i6zLoCNz#sUcY)hJ zFDL3gcl0ltBa=&31BP3D6nDVQxhI}pXodPg;X`RGJ>A(V47vLS8ddwEJiZAxW?EPa z{ow0+gPLl$tBAw#o9Wo%$LSWD4{wN%LAC0^Rfa_di!9?P_V2&xfBQ$`3{iYO$toAe z+ji^Yl&3z)oLzaYM`#gs-)HLHU94sv5$hj6Ym-|l7p6qLddq!CTNQ#9wC;V!H4HgP zDhE0QuSAv)W%F5H{&RF~N)TQf@Y$+M`|LY<-*)?EQ~#v*OY>yMM6nNIi)#4l)JuOA z*i;y%*9WD)g)1=6aMWm{k1_u?N?4_B-5+Zcu3fBq!aJYS|4F$FAMdTb5h zLTfJE>)v%yRFvg-u66v-RF`LQMgQ8E_lz%z9mjI&RWsK*WHOf$83-Ew(^5BG2&6jv z;#ZG{=d?1<=oHMB<|P1=3mHmg*^=-;QqNJ_u2Qe;Ed3jie1@w>FYR@d`MD^VS${k= zkM19D!)}F`zEouBUuu{-C>-YpJHX;LQFESl@;@$a<2()oU#FK?7`w9=4q!6(W<*5Tdn+W`rRpLqCMav2FwX7+ovWQDGKTNBG&iEk>r}ZEP zO13T#eeVz6o2O3p#F96!$Lt{VV~F=B`>=O4_n@Hf$f$ z8(C+MN@f6&a3V1?Mn1&F7gPuBUsimdE5Yyc6<*XH-8|tQDdBEyv=HI~+0LR^eQX?a(@ZJ&RsD>jvvnPUF)+~em(H2Rx5EtmP9>3AmENt+J zCmD{ljs%1g>d% zfv1ncqWf=G+H3Fc9jA?W13bf(Mq1GGXxqB|m-znmhx53=+ABlHjh3^ed#&#bubvQ$ zzr`?)HlP{^@v2H=6w0bQDwIvDftfHECO90F^g8+lGoInK*|=*hXy3HAC~_$YTMoUT zl-fGjbOu>t*G~48_NH(-tBk(?kr=*f0nZIP?jY&fK%?&MhYzt=%{-C8^;C}38}03u zN#e`mv;pDNEg;+O<9(Wkiw@pc_8wJg&Cl4!%Uv1psvsfM?nyQI?9Fx!~RJ!G>J7MJ^&) zif}aJ7uhy49RAh!LxhBL{ZL`*C{;`=XICb6B_gYudTzpEBz`I-`xMK`L3H$b5Xb_F zVynYCBT8F^y#3j(YWkz+odC9t|IiG#nXE=hWv1(-EGDm0;j_d4ORqXA@{rT zh{SETCXMiK;q~cp!q6PUGt1r#VB))^qfOdT5S1LJD|tXsssCOfC1v1SwA)zo=_*?T7tzISXw6?Rhf0;(n{<4Bz|oZBD8Ch>Em0XapAsh;Gw#^_k0d zKXCs0ireabFuwV_{kN0RmrFC9A8NlOPoIKK+oE2R5Yo|!R%9gWCA=Xz94VfMDW5f3 z^h`lZ+^61c`Tkoeco2MaQz@L>oon}TH(tRgYbyhU)UNu3nh5>Zw~@IDohPaHwD=Q_x3qJL;#se#VR41*x*t7{$WRP5|J=?YAVYlK^|=>pPnLqG!KKo^4{an3De{#b&;b76HPdl`n>+7N3otayP;UjQb)#GH*|JnnQ zmTbWy>DznD4@36@MQDrBF1>q5MxrQ*jnT!7P(MQTOTsb5RClkJdIH7zg7D1gAHLzf zXvM0&25*wK<9{e%K&XWcjdFPuSiEAqKB&O3!sa3*GK_+hrs##z>)BEh%HEB+!=QJk4-3Pp`v| zLhriT#<6psO|?~nk};gJf2vc}F|&&D^CA<5iDWtPj*YF#<2HGT2Ocyh>`pBd%kS3% z6j6_J_?V5m&@-#VkK!cA9e!SbI6@XA7&eRNAv1ckRl z){=fSVia#>wWU&ku1A)yY~C$CQO_$&){G933kf4jq>Om~xT8yHT>M=hA2MaDz0zC> ziS@Neu-kE#R{w@N-j*BX#@|-b8vEgAVf_2O{ayeqXDs3uk}5uLidJ>RyrUA6t&dH$ zULzPX8>Me`w18B~Ruuf~2z7CmA>ks{&#LPZZ@=>*im?r4k1+PH<*mLW+nj4!myzCT zv_d^e0=ZB*S`wPp1AIi13l+1^v;MJ&ep#~3oLq@tc~gPxM{!m}l1R49^*->UrH0!2 zfK^KZA$S_D@U_e5^tfM>T%IpI>_k-wVfA|Lb-xfXe7P2&l|@)H zt0;Ez#xOi?XpKp)HV3x-`ZxOyzIY!wC4m~zO%3oNunJ&iPnht0QFZ2?p*LI!5`=<8>VrL|x|Ha<_c_d+IFT2OV?ixv}{QDSr7}VnqO8E}9+M@SfKj ziBAG0;ymC2i5kZpEPrx@ox@ySXzUK?+58Y%x(&MToh9okHEw6R$T+;$Xdt%t`ai99 z-y;##cpevPsa=q6;{Wd21epfTa`!f5QNy=6%sW|#*`bXQt%7DGFfOz)5tY6Kq$XZi z9B4=?h$IBg9>_yv5Lsoh`4u4ogqtHj4hRFW5eXu~z1>RyReXKNL&hTU-V}#68MrQFjHT#n`xD`J zNn;gml&E+OjUdn=lw2)e+6aksPHgUc)mCgIvRiq{)_ho1I{2Z7YkAHYu*ji9`mw3zspWkN=n}(+E(r(r2P4y z_+~KKVxeKJvIf({vUTlPU;dsRiGAPn#w;Go2v)7X`!=(+XW&OgYeiApJsf3+Ps|w4 zyBDnYEs?YhuLRw()bR;Gki|kY`JX>%?n!!y(Zczt5d%Am_wvmNd_vWG=5Y zi`>MOex$}sfWx9X8G}^pC))dY_lfz$#IL0WBfY{ z_tr~cqjmFjOQ`?g9_zaTK9g&UsvBX@n zABCQEwSE4M-LK`2HOdDpYXJvr2#H*()l*7fhdz!kbo>Gm4&`9uTd*BgR9Q}SO@Qi`5h<_7cJZSgMc-gY@}ZrFf!)@%kU zY!SNxBrj!qAm`4vbOJ}Z1VUqtZz)>+92$5|X8p2nilv}yv{bh-4Du=A^;c3-1! zyNQH{gVwk-)88B`0`*8HlfUh_gP-wt{S#As^+%xV_NK9`Z;CSCSCLJ`)9zPsO-t5K|% zUkQ5|aa5FCBL#pr;-it&gV#i1!60bBN?52Hs}WtH^1PaD-+nI! zL~Qq@vvM}u41N?eZOnNNzrfwkmORR>S6s8nb&eNTZ$>PdJ}YWY5Y}H6cWTUe759&q zLa+Vqy<#{mjM(?b@0#zT;b}+f(^o3L(1tgX3KImg+`bU{l+>dI+OYZfoo1-5QI;*( zSWhdg4>q9j>Tkx6XaTU><0{&8jwlgn(2=8LKI9;W#^gJ_D*jY}({z{+G%S=qO1F8< zhPbi00exk|xBMRo#*JJ@I&!c|G;^s3F|e^^h{y1!fGj%2n7SsBhu&laN(1<}cy+dm zGI}=wQBkNL%6`TIc3}LdaqISN+Rc$|!X(*Sz8*?S{%I5EVQj2mz#v8JQv^_yl7k}hH)mQ0|U+)L+Kd2%?ol%2IZ&pSsaaddg!(~Qls zXd%1WQaS;&^1GEb+C$|}4CtPCdtLI~G+#lD{@++L)4lx0=#P~%6~@?i9c1Y*L%Oef zQ6&4*+W#Bk%EO`P!g#C&=V%O>Ak{HDT;i*#qyg-{CE9VHNRX!58kPG@;)ud)Ph5R) z(Vl!RCL)r3CAsi=b$#1699rqEPvhm_hur#M_q^w(9X^YZq?L=*>8pbz;4g)gfO>%G zjL5njV>h}YDttf~XPvY-p(qLXvh}n36q7O*XD!|!TSGQR(qrEL*Dt56&pu5{>X#Ps zjX7e+9cfMsfVU9L8MRv{vQf#uc>)2}Oi0*=URZVZY_dzP07B8~Zq4!(pfZ#e^|>{kfcI>!AO|Ix%UXZLLvlxlt%4UBbS? z$#;&rQNFa*D?RkL(bW`dTW$8gr_`nQL^p9!g;@92l$|%MhJpWp?I;mm*_Nd#LvP+0 z^TWOqmFcnxDHVN2a0bvs?L!ztl*wSFAPQFQVg-W4kHpJW3D7#kD<(VfyRVQ?M$n&@UhEl_=2)bX`5?8icj8EF%WOl2e|C6YSLJTNLSit$5N0WsDO1suvv8ocZIU4C(lq0Wtp$H%r86EtPs zCRFf{3{`Ee!XW0eCDt11y6az%K|L1FDA?XiLoy(#RUcoD9sMnZV2xva&p#4CtCXz# zea&3Ls$6nYUm&jR`THzW*S~k5ITrQcUE+L=;sry;x){Etr48qfkd_{OR~pIy5``?0 z=S@@sx}e8l+(jmPq`lAe&FbyQ4?EMbywK_3H1c_+O0=u$^`eYQJfsV@0=M^=VQ6>b zw$=w4NRqq!xu6&hadY-v%pd^wsyVdKov{AVP zKhq4jRLOxk<6q(V0Qk8JuqJ6JVFEz{J`V9uga!B0k;fL`h#tY=a4l+lr?#dXLk#lR z^xn-IXVBeL;Q-l3rX`*5y)vd|(F4H`jl$CAezD2?erB@4|2i6%Nc~i0Jd2jVC?sjk zyarij9y%j-E+cbfxBqf6(Nq2?K)Qp?c0l}Rj(wld3{hl451XZ z(zfG5+5Pm5L-674-D$w-!UGfy*(u+9wJ{O4rrL%YiB0Z;T%^^ABfKwS;`!F4kjz#? za&wgi^p8AOG$_>jv>~EJzimA0lB?a$_H~=m>bE)+$iZFdm?Y&U(n3Umb?irKYOldS zn@}%ozeuD&4x=}pRty6Uj(IW(n}6MG%_{uCD0|6?;9=7sF5}y#$=6F8$99unKl^q) zE<9g8`w#&*M9fMgg8q3?0?k9IzwX;V9FZ1oI#jnd;jI4f8Fiw(yBQZg&(d1yyzTS3 zAC;nyXA3L((RbB1M-TP~endiEv6go_4wCIkqL<7G#kqfrC!V)_Wf$N+P61eTyslUZ zJ=KsyzDB|HEAW3nK`-ZC!lyevTdoWPNQX&3ul0?W`y>dhG_cZiC`PjTZ5?|D^ z^|4iiuBakvwth}w1P_fvq^`M9x@cwkmtJ1|J)yym4pz|6z8bAzLt?QS*>On%%_4!w zWSUvYeo;x699Vd0MbXs9uK%s@uE1p+Pt*qMz!AI^b0B>Xot38i*ZMnX_4VOc;xCeV zPU}v9fZc6NW3SK7FT-|2ViDn2v+;aiaR8ZTF*8a)ZA0j5tk{13V-fz%D{kAP$=40% zRZ|CGB8B6Z-!DbIcT?5#m*mb@8_rf2t|Y9l zUxPn?Ip6{-Y1v%~Z%=$OE`A%10o;f}Mm1b2SF8EEgj)@?f{@iR!IR_A^&Hv~+ZIr# z{(Pl%Ixus!mPe{*YN)Kd2G7Je9{ypO2I)my)1mrE_{<2)?x=I%e8q(fPt^;!{Qdgo+Mo}Qrszh|-C#vEsiggot;M&zOt zIj%f4;dOLj3zw0umX8$7whui6CcaP1@p~Kd1cu6n4nThCDjN=D*?i+8REP4m#Q1Q@oo0m4~~hIHOg=C6B^bFQc{ zU-@NzoP_JT#(m_-ey_81>A~YD-dRAvD~9io5j-DJ>0)#b1K0Fs*GFk$d3PkvSTznn zHqh0UfGmb>Mly^{6VODrGAaSkP(7_7?w7fC6cT#t`&YvC8@)_*56%zNF{&}V!Dk;ViQ8C?LtL*#G$fg zu7xAHTX{+0`|5abCAKsgy9yPKbE+$dB1}yQ#|UMNwvwQ>K8K1_ofm_edkSouGga;| z@{N$_{G50^&HJCCENWXTt0xgU>^_N%ftTet?F?5-4FB z4emR5>^CBx19MspYRjZvhd5Radg83GZ=b7q+MOj_*w3l%*7er#3T%LTqLVmyL6_|( zb@SZdpIvm9UjuaedLO;pE*^}QxEYTd{X`4?)d_#)Xg8kpl18Wh|5*Sw2l}#>WJ$mM z3gw*#qG$&cCa>CF)Hm12?1+|mY90R%0M$S$zaM_|k#*^IvTmMv?4f(4f?o*!((96| z?BPcqlhEVV8?UqV*Ir%2g4cZB^T0#+Qs3HoUw@6YTBRj6&Sg4=GwkbcaJ*v^*{k4q z84o)V`|J;)1q4HP!4T*6+jQb^5)B{*uvTc?z|OhoD(ijg z9T1cpX4hSIp7!0h-uZxCe&sdLtY~3ZU3$Ll0JU(ApL-sC+OC37V0PI|7{tB7w#1Dp zM?oy9Ajed&MJv)3i!!$F85Km!i^qyXiga7HhC^sy)G_%2=Sf}Y(Wu$c{^$~)HMf=h4&1A?3rjf7-r zpJUIx@H}i)@TzER?YRCxy|pGnkIb4g+tyjTtv&nL12Don3zIzKp`8-=@Phd7T+m7V zdeF3a<1IV0#|6^FdEo9_ZT+=YM;-A|05>6iFmND_Z~P57F2;turUYWU?0JBVhmhgH zdvCYRH-?Q8q~ltJhFHJ&VyHE4+Q8a?4rk#jc^wFSZoTheyWpbBZF2}3pG2O?JTe9i zpCh`SVlV&iHE2hi2Vp~J2n}$v4QPD~j>PS{XJ-jFpL_B_YtyQ=1ZsQid#L?1@+aJq zgRy(%MOKI-QZz(*-7U94JEyhnyXWqj9C-AF8ZSOqg2IUh;EPAC>LAa1?+>&SPdiJ3 zqUWD_6k}#3_3e0QXukL1VB7d_>qAgkB)U51yi4rKC!e*yZ@P)y4K1rCpc}3~``rDY zU5V>PusXYZ!C~ z@InWEFAR|b_yn8HzY#Z%AxBd{n9xKz5{ZGUF)dz*w8X`<2-Eu8p9>m)Y5ZeY!wwEK zN(S)g&8y-88s8|A8fn`~P=cd35KyubRSjSC!0M_k{wpuWO&||E5zbdUGWbLbkWoeb zpW(l1*PjxUIN_3*In+{pg??L9$9#0kf*0O;Lq@88=irdb!QkjfgcA{{m_bGOoSx!9 zMj_0{2YQMs=Cn7cJHr0$RLrWN?7BL~qOyi$%Oy+50yazJE9daQ} zZgJ*?dRfx0ArLwh7QM?h{McsFJQq7PA-tMhvw_uE`$8*jh}(~9@=QInNw9e2qKvf3 zBQbOuCoCX^4@IPY0>%RVL&O5or_*|W!kFt+U-E)(S8ZV}`#a#n~u!b4@>WP1DE~)YH z4YWr71?`49u%+VI3E3d{TX+sD0s-60yRoX z?C}ThhMkBe5@7Vgk)DShe$);)V1K(Angc&$Gg%*YAwK_Vm>mnj3i;3NeXh5S)?W`B zCme~urnPQyZTn@+cx;kpW60v>5m+5P{j3Y@t$sLya`mOQ$8P_W?S~QH{$TqYbQr!a z*R&U&eAJpZZwmVdvuvfN4ehK8u7pO(op$8mhuBRJEKSA+|DwyUv3u{oAMM=7ZoA9fL6f=b_Fz2x*T?({qpO0w6y{3 zWPD&p9&?f1Uz|fA92^8~q>zAXg%W6pJoW5=|`tnNK8Z*(}ZYP~?15uCjdYom)9)19{2Vk%iM^75Vpel`zPJyNf&-*uNRL^?f z{s8P{+-3)%zL%cgL#*tzH{P~0AjoNeV>x%f{qnJ=0!$^G(}tuf6Q* z%g@Kg9__(#iUH6Z=ypnX31n`%`U>l?*+%x_YyIrh?q}n8MI*cYmTR%GUO|RDX~*QL z=bpFAuf7i2EgM=NXc5#!-6#a|<2GzNh48gt*JTvyKS7!bl2Z#wfo}pQr0@E5>*1>2 zm4qA4$vp1C5`;IE6qqaM08v_h@dg%ZS$+Q1aO-mX$#S)H-F>^}22Khm{NGvj#+z?hPiSQvcho`H*iXfgsuBn^$Jjq1jG-3H zGZ6l)1udk{Ae=m+%LzIv^zgm6;RxEgup=|Y3ZY3!0S%7}&4nE|+O2u%)z=^(fObx# zC$yi(Lbu+kz5Q+7^>k#a^C3qev<8lp9d5hrvZD;amOy)_B{aC$-s#XVX$nE%9{V3@ z@4ow=+2Ty5l(B6ubGl%W?FmgMAL|u{|Luse=vgGmk!K zt3X&Z1LKHmM{01?hmhsJF|za39YF6@4N-w2+ab_wJk6w&kh zopI|8?Kf(D8Tt~<@ii0Da!iqDWb3bN{kgoazp*Fgk|*sr9mWt%js6~j)3MCu@AF5} zMhU+_=nNv$(a&;~6hD|edNCKywlPjM;!E0jofyR%FjyL0;^S*F%E`=;AH6F< z$xsMNz#NLPproLLP}xRh2~U>gPL)Xc=|##AoQ`FTzb_z?3Fx>OLtUt{{r14GR2ONt zv}?;K)kDlmiPI41r(1mZTp)4NE!A3b$3W?Pryqo0A*uZS~q50m(mXl3>xu%?Z_5osOA~okr*VPr-@M=Yjh5_Yhh!GNwa#GX7Gz z*g2#>!W{&SlJWQo&Ay?H652@cqc_C+;?rLQ9l zHG3~?DlhKU1GX$mVEg219CvwGf(?E_rf{Zdvt~Gkv#A|^=m9t?(A*7qBJCbf(H{&O z2EA{();dEY<=3&3^yQa!4S3wA7=nN&9=ykzL2ID^w^{Urt&m3_eayNXdkk!VoD3UY z)1d(XK?n@Y9(Y(6)#uJWH^R8)2C$DZOLjRPd+J&152KEwvC*D39k;q*?9Q4!Q(xz= zxa1<+XSZE-WPtVz_CkFqDEZGbkK0O6^`C-G^NKtYa3M5GZoeH`4~IceavpBTn1Xg* zZhc?)k99ltXglYOldxIz8(_FWJo)q<_7aSSo^rzRc6`^Pupul&*+*Fa{_oq>S6*gE z9JrUX(P%L9z5WAntl?N(N8G|*?)!v}K9S!_AOgOo8Nb5#hmrz16oL|LsGsipA3O7$ zb8Yi~`~x=?U8DTuf!yRDck&t5@9np(SI@KUxFZg<8(|7c$oF6E_l})*2JD}-Y6Y7SSK10VMlu%y7$C_my7X#$?5U^h z2%sG+~Onsc-8i_aa zG=yED*WY{_+8Nz->kQ}P_#ubK7tT5JG&||IV|2{sI0(DmLRuR0r3MNOw(=O%PTOs3 z2kyJK)q#+)7zXqA*#B@FI&3K5HiY(BEnkUZeMTEQ6cBN3df~Y}?a=-IWgmY&%=W_7 z$uye${L_y~$XNzq57%GRPA2`8qaIx#OnVFTb?S*H*tut%XvcT&Y0ve20YcDy?c$zi zsju&cJ)*0wzCmc))_3VD4 zYv*8&CjYO+^*8eAV;tT^8T?@+=u9DaWb-dZ*HT)4aIZmungk^;;I+K{~7+Pmj0ko!l@^0z&JL1Jfug6 zhD;^Fd1)%X@aPQ(8yN*R$r_{xAs>~PA~4jr(dJ%WX(FR#fuTPB@G_WAb(DvG)-1-U z5MoG79K2kU}!dFf3QwjVS#qltC9t;gO-qGr{+S`gs>Y>fRFBUGk zju3tx!6ChfW=K0Ye!}BZy1p+Kv>4A+bs};Tgt#!IrVae!8=Q;B^}tZGj)D>p4AOp=d&Zb{4Ar6sx#Sfn-duj2X41V>+ zzpetsM)uP$zO;i5?}9JorPlX}hj6UL1tkq|{D8(C&+2);t-1Q@cFT=d*&3^@Y?oYf zi`~-uHXNZ?8Jqd$+Kh>OomgxZz83HH&z={UxyJ@Y{an%rlD3E zqU_kQz1@JXzoj@{L*uMGjxuQAM|KMZC41~*qp4AX;|L!P`oxaG*Zoo$k$emqB`Z)+ z0>hu@UVMc;{OF_B_1L4V*V(6Ob6N~L9Y6mv5_S)|K$u@^=MqMmDG8Sj)N4 zczFdItzy)_r8U598JrhJjU1^>)>W5XB-;|y0{9LFR=L6B{QA;!PuL30n_;6l%~ovM z2!fI;?Uq~ba*dKJ&&OtVqSXAK|IbS}s&cHI-Tin8ZMX@njg8r<=bVS*G%rDb&=qzb zj*;yoZZ_Y?jW5?+d5Ily&^{0fO@xh@hPVZ0fCMEiTC|j)L`Eesz49Xkc?X^513u|N z80SDT6&?k7y6=n7ROn@!ZMv!KeDLUy)+gXc6j=0o=N)O39CHNhj^H@W9rrv41D-dr zSP21|L5Rhp9^=N2vmbu^(f04Wk6nI2Pt2z{E(IGU-Oug`dnPMmGkP7;H`B2bravFr zJ&!;2jGfrEi=B2-SDQ9-2HIJQTZV>PSHR`1M7^)S!a8o&&YpVye<ZprF7kS6e7yL-*XIo zWy_Y(-l~IfFxhy+)B5YJi{m=I(WicU&2d3c(hG)+k3Z%x8-wFsJW@3hS|R_$(Hqw7 z*+=fzjZ3e;^&T`(&eY90cl5r&R)?`%9y8*+!(-ZP7eB21jAM0uUwp-8LaU0x{Mc-2 zr|htGC);3y^)P?Ug?*rjcHcvfghe!U2HQE;^(JVyplq*2XisG&Eg^hP;M!aRKCi>m zq7Trx5F$=p+k}UfYT~OQ3}=@83N4=*KpkuC8(41EwU`XG%stRA_(!~}dlkWaaG_uP zg<)I>U_nX&OGu(P_+u9&O=NsPMeut62l`uP;Bx^U7BtkVO;JZxMbX4vBbD8nWj(md$Z62!ax-kTZP; z+9;7miN~P=-cfig>iCa9770EQ69fe&lW+!_c;mvHbm)d`E0n@U#EQf^DT$}c{j$|4 zP6Uef3wO%w`9mc|TJyD$_*!!Q!~Ubo;8#EVb57w+CSQN^jqM3L2X*ij`9IG+4nYa7 zJq7(X$MJ&u9(od*5|?QsMFXvCuGYe?#?hNgFTK)s+=)kTuCp=YCSlC`ms#E{!y9d8 z&MZqb8JaYxD;oOl=TTC*=f>-u_dk>|)QK?mdG8(A2tm7q{CUp<4zt072U{Q5veZEDx!dGw~(9>dKT%_S(QWxX!C(jIyQf|4#r+Xd&Grt~Exg|dOcjq4l; zN^axEb<>UQzi(l~dfM6gvfUG#&@DFI1n1o6;OI>Ov>VQc{S6$wx#}|8`(L{t-~Y3B z7Jyn5T^~QdB`z(ZfP{g8@fsK?A|lvb*a4V;f`mv(DF(Kn*kFMeD0aQRHU=2jluD>b zcfWvhU zHaj0>u8921g7dW3^*7n0j}5kSQDP}eI1)$5;RaMXG|qx(^ZxMR)}>;CuccdhMJ5K3y)aBh{#h|EYysEAhGAzF!0 zBBh^gF25AQ)It|K`fkh^Yu+*tO8UZ;bWoi=?IlESS|NHPbJGvN2_fR#5_F1ajw@cG zC`7x`wh%5Ktso+>YYtWWMcflCHb=k2c^9{aOXE4W3?vQ{Eei4UrwMB@?WncWj`rqD z&%kXUF)Q+*o_c&0DPaEfi<4-X*E(OJG~z47|8W9Vk|6jkgj-|SE3YE%HH3aGV*~p4 zvs#G2OoA|{KAQ=_ zM5Jm-^rk=kQ^dJS)F1x?&R0Vul6aJKgvdh96gWymC=t;|qBkmcmW^roAgd)moxR3T<20t9`v8Qd*y3Y&v$X^327 z$Ngj{G(H<5UU4`|2HTbuA+A9PbDcZ%w}>^;l{9hEFX$hUxZZp_>WIT!PfN}1wgrc8 zbEW9Tus}8JMJzgX>4u5x9TB~`i*~CgsX2clsO8W`H^0ogD58WN;V5}}=(Bc3%jR}f z+ZHx`=3EztT3!F=-+yP3+^JfNk2qXrBI8hUtgT0Dr~hBYL?bU;9Il(yP}B=zry#Cws->N#kGM2p)f3mLFHy zYw4SFif0d`ja(t%D_L<7o{B&qfz`=dQiR)JZDfVhy3NwW9N!oT=DNjFe`S>YhGqpw z1otYxX*fh+$Qvh0&CAJTlr!?%an4Sz=x+6f+vTt+(jjBjXI+&q_ICr?K7=0d;WzU0DY*71rQ$@R!3XT^hfa}E=R?pw z{P=^gzj+59=ZWP1>(VHBR{j-Aj+K6_ym+vqMxC3DI0-R8j0RWv_u>w6*u3^ zb*gklsp=pIB^N^|=>|v1OgI1}&LJx^XTl}37?I5W5`{qA*?_I*eX&c{_XiJr3azY+77!j(z$i)=aKwZ?gRH%;15x({@{9DtD=s zhFHzHLVx z+R!cH=n$`E$aS=^kc&`IibMDFFTZY|p@dd?XJoiHtf&A1AZfTt*<+3BRqgasPhd{Q zgU1h3nj_aR|7qWl19B{YDDv`aZ`poWlGzP0m$-hT$4#(P&uI$LZ-EWC|8A>;>1Pp2 z&bgpD1cnLr@I(ErA=Z~<2}ztWmm=~at1Nv{*n85^ha$d(SR3_!2qI*s?!9cE+O_TB z2l{#aMa+<{msehW%{pDx&f2xO(5BCt4-qp5OE}XZl$`GC%X3dYiizTF?DuIi?D&(< zfeD zJ8TO#R_Kqfz8_~NpM}*Lh)mB65`ktLyQozMd+qhtF}u5`4M10oI6{=K;QRy@X|{oo zCUG~dG2;<8Iu1gt%q_q8^kBEdBV15lI2R`A1apns3n7Mnf{Aijpb~e8+SGyhXc`14 z5gbQ;{*|50*pN6=Hwa1RHa-zOM=M+z?I)jOo#*Rsu}pO|ITpgND7t!j*-(gStuAX} z9l$Y(Nclz18v38asYlS^*#zhR{}1cG_VKw8N`@nP8O#HVAWwJDEh3aOtPi2&U^q%3 ziiW*q!0v$L>bvM$SRVfH!7tnY+5I2*@UPqdvXRj6@j1=MM&t}0VYpJY)=AveX4#|`$-UpVcD2W`t$mh*aG5o;q zc`{?j>dhiH4W+qNI}r208tlqEv@26yLG4L}Qk&>2_Ke?2yJJTPh=XI%8*!9K^u{gl z-#!pm-X1RhsDFHdm3N*r8Tb>8pb%eO z6P{@M=7IPqK;?LZe|GsYy(|ieH-)yzp>ZIQ8a30&AsmF9Xu=^$-|8Q4v5;OaB8ojW zH99@5XBOodqoY#feL{rQMvDVTB>YqPB9zRS2}cQ{HzJgXqa^M}Z-*BUC5+R@@4SV3 zI@L1{j^9bNJRbBV-)gUPl+2RojZR`be`ov)BZb2oYyQcWzv;qX)|h-cVqkRW($ii- z7emF0747Jwj^Ji`Yx@92qwk|bLwLP2H}iMga-CH|EMq1*ESg??8Hyyov6|c0gvikV zrJmL7d(5Q1_WGMHUek|T^~P{!+%sU1UEQOXZL{sR)(BI;yTjtIL;ss^y^A=*2M}tC z+vDh=*qK{(J?C=R{FF`cF12de;aJEyv|$6=1#2lEBaU-PtG0~iQuf%u`%&yz!Efff zqd;}=6GQBx3(vQ%?XR#|D3KKLe)6wV+*l(XbFjM=TUC_KWqCpGoxMPp9 z!^m4)Z;!&x|G|eJ`P$LDXIDG%*rRwd_daAz;mWuLlfSd)FM#0mI~IPfa4Q|VV+oTKF95q91MmqIA1VlNJTBH~7imJR`lImV}+ z)chluY73N$NXoP5mFb95-8%d3<3u8vyVZeIT$wtAzx)1s*7d3@G5LF@{mMc6^0uAq ztbH@7wmwbI&2FFwahFd=i#QIx@qWz5ULI*1mH$I7+J7-4K1Y zL~qN|C5xSVs7LRc?Wt#;MfB)G#4av!LW4M5M#Hhx0&x;?5%jyI58O#RSuyI;wo^}g zYUog_T)DEFjNNCiJvi{sfYag~KP+E-VH3L=Jssj|@R`)tme782LzRPoR#eNlKWSj# z5l!~2U=RRyv=99-Y3pwHym8*jA-2R>r8_uj)!J>dk~ z5ox0qV{UP3PY8Dp-iPQD1Fa$!U@p8Iy(lle%vjmYjy$}PH9Dk$)!KeLtWZs{ z6HYzL#b89#+NQ?VPC&Z#&i;1K{SR7sh-}9keYlU+FBzN9Anqnkoo9zUVw+)(SzI15 z@jl&t?VrlS_0_&}PkZFiL6{ic3p_v2MSEmM_XR|DK1U(%Hrs9kfpLJ9rH=0~c3V=< zWnAFZrr!>!U(X30(#7)nYj0W=M4p}+^pI7lT*-0&#M94%$eA07_5m-}NXKLPNjg`w zMvJ@cCG_O9gUBbMN(ttM+fn|zPi@5J;Os66@kAUwE!%X26KgmeZh#Olo$#I-u2bESzZpX~<6*|X0Lb1o+lTxLoq591--e!<=MEbPk?#BrB)e4Fn4 zuX#OeL;tz$7ix$b>SKxCprgczA=-RWd5LVLt=?$==i}bnmFdgu{%h^epIQ9hw8npi zqa+?5@tX;6bx(-0`iKgd6^Y)2MHU|oQf*U`WZ_$?WaaVzAILbPs1jH4W1N!Vr=;<3 zNJCTtTnCCGMClYyht+VCVl0$XUG90SEjPo=TxAaZ+6FK^uz^Ub6FcI!idxMRqO3k* z{t&;_;%Y}5iJ;6iHV-TdJgP}N;Zk7!#0AtLL|J{rI7G{p=f{sGcYVYxm-=?j68W>s zkz0?;Gz->52;ii4)Z3GLA9^&ev--3U3P6Gmn&7Wr#Z97mLiY(F-cOnDmD~`zytur= z%$)S2@(9OxZ%L>F(ewyr|K#!o1s(oFr8%&(!PTRKyNlL17YX6KWl=!DR($^lfh4#+ z7d11_6QCEvFi((wtQwNvC-W`*qqrb?GY1_de7kvI5lVz4`3t9nkUIRXT6!Fw4#^)k z^Bv>AGIh}#bQ{Hl67}&)#wfI#zdZhf4g%GI#wG1gM#0Pdl|T3xz)OJI8i)Jm+Rs1P z@&!men;!gq4rNX(!-4Z-tO~U2+?5;giL^CF9>ZctRTn|o0ao|H5H!l6Lqobm{`c+& z)}!|g_7i4*#Yt9#oAAZluuCVyF-ISPj*1o>^efxDAAD?gpm*S_Z@zQ=4y7ndbMhiY zSjwPevgM^0aIkJl*diCJdHUJst=Dxoa=4x61aOHTw7IkyN<=@h^DnrB!&WJK`FX^H z5ZTZMx(!?p4?OY+91Jb(`koz8lsU`wbo_#t&(SBJ?qUUl2j1t0IkmTQ&)$BR7Z*uM z^ke8P9Tz1_l=Q<^KTHXqf$~xvh>LK8-n-AOi1s|;B06Q^pc(k!J$CT^d)sAKc1OhG zZcO4{VE18}Wil3Sw&cM4B=x_oUw@l8VG{GBI6;9WYf|-Xw!y@12RppsL2wAnL3F6R zH8|=N2vHxpu8&L3!|DxIVk%as0L(t*2L5cTQLVbY|G$@ogT=`R(d z^-sz-pTHcd6n7q6udWRmc(02@>ZV^e>=JEw6GfpnUfaW3T-3zdHR!45?7ACnMfvPZ z`k<88SJ_qAyrPXCKi1AX<237g^L4(qzW(MrST$*F+t8lJ2K0w;hUg72EQ(mfl@L${ zV1cMpyVlmRO|TXt4xsNwk46X1CFm7fYL5WFz4qAE&3R5i=SEu&#h-ogg)LjW3`;E~ zToI}CTkN~fKGvaaYb@XF>|7Z-1o^b)O%Je;7%pf0Xu8w<7uHwB%b+lv1yECT;L@_J zMIt~i{3@!Rc>g78koHy5^)mGNm+k(C2HKdhI5&&*ph&duQ^fE(ckgYZ$Bg#zC1t?kh!Sz7e!Y4qQf_V4(O)8wv&-9bvd5l! z(mJ=xpk$HTY<1^Jzxl@D)A4pM* zc)<+uqY*{~S-tTm_6Ew4}VNf-`I`pP(=PUkN_WEbLuu`EsvKjwHoEcOg^DFD?I zvg4aDEnc~>#Fj#qTpl_Yvc~zXLdYr3+f(x68AhIw`!BXXPmZc5F|ucm9feXy{bx(R(|~IFFa%zW&KYsUu{-PoEV4O z<%**g#FlM1C;4G1CJd-yVy4jTm0%P4P6`E?t|2ISy9I~1{TU~&I3RvUh2?uCq3)oD zywxAlQ6g65LWwe0L~p#0)bB+48~C5DLjLmAm>5Uh>J1w=4a__fotGcy68c{tSkw4V zfTV~?p*U%iHU^G6Nf-_RLV#UI^G~*X0XKkjQ-r@JNm-zfQq57{d}G6&f7O1QHq}aS znAm0Ko$Nom?ur$bPq>jTXD1zhjB{&9xWo2#%n^qI zu$AaISn8r6^XJX8mk?nX180HElxkj(dEdHq_q9U~tcxxRbW`Y`FKEWC@*DQeH$S*i z%Hy~p-+$k|?B_|pa&Uc#16hupdq!gy^(aOi-$J?Q$DfX{2KDP&-2?U`4F@m|g$owW zx5u9xs>#PrKH*sBlF-IlW^$jx`iKZObLP!ObYyea<)H)8r|3pG^~B?7LruTAF2Uh{ zCWpXbh_XzijX6b%+4*NTwi>kaH7tXC2p7tMb@sQz8y$$^Qr*_WCeESlTg;)p^2QrB z{>KSU%&7!d!7)c3YJ0#@GIy@#rb2#OKIo~T#6b_raYtdEwHA3mEP@+mB1E1+L!Ndq zob%2;9sLbh5`*ZHd9;#f`~zjye;@1A-{(0zzyD#JTi@A!```qHj1Lj;VFpkUjU<&9_`semEl$q5g^2oJ{`(Z~*fI5!jh z8Ur7D!pF)v=(H(cz8nM`2z}IP2Am(#O*3Z9I9JdsBITZlz8rOE0~ggpbG>P@jmqYmfiz3Xw!2rNTK8pmM0^f1PmgJ48Os z5tS=#X>}3#sE6Ul7Gw=7f>0QT}^8^M!Cy^(;o+-}vwU z5Z4)Hhr-=c2i-V}79yen7g^l+xO}gPKEUPl{yRZ94-3iob)3(xzw3Yhp*_6L7{f<= ziEbd2xq9CP_=~Ho;X%y5haBKKN__vB-Twq6&F=rA!K(eRo@fBapV8#^ZyW!pPGE#2 z$PqwO(68Vm;N*Q`-}s#gm5vhMfG~lJco8g*gdCDFspu*mZykuW=%_B4TV8!D4u8D0 z%?WYvivv7eRKCZ_#8>$qpIuQdit2LNAo`(!t*FNMeMH2ec`Dh9p^;7=KN-{RmM;2m;Rv2g*OR#ET}8oDNSB8Qmz zCTg6YVkK_H2pRp8%J)8x;RS4cvzlsuP_9e8#VNr?(YGGF)&2lmZR8{Bze9l#`YyDT zCg*u#-}s#;5U_`|f|K%(VXDkRfADtvgQMizAKg5#i{9)}%VpWsPeL0#t0{qyOh0(U z)X$$UU$|J9!^c9K&*4uuxbYJo0z-XOaYDRKsd8h4!pnmc4M$x5#+-kA{87J+HUDMH z*SxZB;qQ1QiYCYy6uXvSQ<~#kl+rDc%T4@36f=~_5keMnZbtFsi729jx5PYhix)xc zWRY`|mEqtkjYYFyZ+CoWCWx(El`oU)LqT!Xmt|SUmM~4v6X8JFQx6xBBz?KEB~3F2 zdMMp$&BBFCAUNcp*MUR1@@GxfBhew5A1+1{7NcxLN*r|>)LbCKLV0eO^~{~mtvCl? zJ=!L0j#=5I#Gi+0;LXaGa{U%7Ka#&{S^LFR}DD%g~ivbrdS)}Wt45ok=lCN}j=#Z?zt-4pB9yo0Q78@j@ zx-(m{j|g!7)${{SP^`f7YCZt`xH&kG?q=0Z?Wt!DstuRrB$ zl@neH4uHkEPnLjlB1V-(xvbhN9au}XzX8v^@2N+*(k0L@6BM&nU<^tB&g?miMaG22 zqxxS!R6puZ8TnR81@R#conkq~=qvDDWq`j$si)T!d|8AQBTlNs40);l%9k&#Y`{_J z5-MrMA&dyO<}((H$aHVM{#W}Y>ZAVjgq|1ep#F}})T<127xoK}=8=!b4ORF47e0u9 zQyM&+&D@~#w>c z?7!Ca`V;p*C8)=||NWhUzt)@L%n=`r9OWD(g_y76`A>RoBzkj%l(}mD)L8I*l%4is zA*S%b%ZmCljsHo&YRb>{|HkeAEd19k{29X<&H?RxU?d-j!@bw^6US6W@uKCMG6t?c ziom0R=1wBHTfNC6-#Dsbc^-5n|VacF_>IA*&2QPv^8R!lv;qn!XylAqrIDBOMci5}Hh zDIER;`UL9}t?w&5%wtea=%bK_l83Y*qdfWYAy9-mDOM`32Pk%U{)y$Y&7gA8<}_-H zZe7*>7>-UnacByIJvwa;ZPj5a-Tv4=w7B5k7=Ekm|2RSc$$j)w;AhTJqRn9;w|Y|& zrFkL0P_F`#CSGy&Yg4{9Z`y?Z0t^25iNCoP|Lv+*;#MoiH>lbOuFd$@d=P+3HB~_# zdLcMzqr2DR{F5zz(}uqeJL0ksYAr|1QwMk*UUmB~oc6g^2P_7K4!nWj2d;yAO6%$I z*CXp0!WZ2lyDkI`B%Oaq@1l^zkr`Q{_#s$%Xbw^NeydDeO*;~AkeL?cyUaReB}ms9hf;K49w?OX0d0ue$aq7&A?FgFIcP%Z~zxz%>1bIVt>Np(^kr87c@ zTycy@?-S4@Z>?>#bvYJCI3-uFvbnWr-_-`7J7V*4JOsYL8$mSUH@j3#}!t>JDsmpeK*GJp?+Q z=zL2PyglOB(YmZYmRV$+P^|_;3K{prf)?j;tuD-g1ZjAu(<} z0fKTw?OzY`pY|02k04HZ1F4e;|5PauaMbT28Yo;t0WhcNVjQ$r<#`SLyU&K6hxa#2TpbqMP z?Q`@OKT1cpqufy0ZE|f$3s7%UN-GmE*xd?RnFH1XWliKd}Ab#jKngc#b zY)C}BfRmbMsD=O$J#qPJzsmRA^$`I@-}>`M{yZw*owUc@>jD3=IKb_E38rd{V{JHtlp%%O1(t)gq~7osjtXPJ*>a@$@YI3e*#e!|2O^d?}U;V z`Std(5&g#gdHgg2HIn||307~2>xmUV3^t!fHL-d&n&|Up7qa(6NnQLvPlCcstYNC< ztvup7L}mDZ&Qmu6_|pwnDQ+_)o9zYqq7f&~`%ae}hvf15lkJbwdb`O$i%>8R3`YqY zBe4n}6cI`kSK)f{l;uA8{QhG3VdD}u1fDlP+E@qnw`Tr^-(|+F^`Zuj+|N6V7IS_!t3~9mjMyam4a3MM&BY zRfUrjpPGO9$4u;FQS*bwqP~*(m(Tc2C-`a32y+Ps7yc<%UpI)SlsY)7*!a~Gx|__H6Ha={%RlpkS;&e6L>gW zJkY_LZhts{s)3<=`Rk$lbpMA#Z4|El zS%>BS5&T!z|7yQ*MuZX7NN|_QV!31dSCi;n$64o0qERvag*B7N#0bO@rR0g5sl*Zd z-Jd?A#~nV5{>n|@LyB;ere_9(zZ}+ zp7yK!0EO`TN9C&>>G7x3;Zb98QI?2&f<_wtYMsh=&K{p9B3L0*+P!d4ngC=n%j=bJ{g+Zk$hz#>a%jhUkDi4%H1&7Hv@l!hd{ErN^ zCy9Q7zvr$vJeu$|-I6AmWaWduHfhSkCu7C+eC5zWi#B#{$R)FvR5}FZ`&FW7(ny?% z!{esE*3|xV#xb$NmAQ`m-bHV6VG%wE^T1(q9NH5qz5dFNVJ8@^(9Me;H7IUW1Vi5o zN1j?^;@}wm@(}z(^i|=XpN!xc)zuAAqE`gh#hw z-b~*N;U`!|?e`5!95rgb;2+=+(nsF@;rYb$)||ky!HmuTHxx^OtuuxqecfyM+qDy zh%Y)9V|M?O-T!6xzpLH<=OdJ4wg(ze@m4|ZRi6;Lx;Qgs^(NYIq$icI1qsJsofSo0 zP9B^IM}l`6+v+?8cm$8`RB5)|e$Wf29Nd27)Q&S`8I`Zgq+B>ka*=-yjSA)G%{xlM zOLf$fr|967`VFCLjuS;ElBy!M(f6gJ1lBiEd+Uo(qEa+TMWv?mOS@AAXEG(tn&V_r zIj2JazNgU%)XmEQ1l>4_^;cXB&dLq!Jt}boBh@jKqYs7Xr8uj?Ut!@!z$58L9!?yP zJ5(X$;lJb)@+Ql+1)s!a?D{y>@g$NvxkHPe#c{{v4Bl0v-8d$K`t& zlrKWbuTvqE0Gpj_Rz<%-Nm~jIWNs0attuR#vpW3qP$8{H4}jg+^G~*X5oUb^M5xYK zpno>h{GWyYzYG3;F~K}0Q6$}@tN+7=M6_mw*&)oEOcBCG=|G9`#BW&htv?|NF|+fJ zgCaZs7QFw-u0NnhcKwZ}i;cJc2!s-G)ggKVM~M?dwAbUGl!`Y*^yWxey@5C!xX-Av zPj3b9zoQ0a=fCXy@2K!^oBtC+i8c#9$%he<`t_p}VW}92GjXKi%ZYL*R~#26iVzrb zS8mQi9OtJjcswVM!eHh@LpoH56dR7!A*BLo?37t=^2p>J9ZN!By8mwRed&aG|ZL z`Knq1%YXm?KmbWZK~ztiI#pKUOdR==73?3l2NYEsM`yZpI7$XsDFS;QT;9S=rzl`ZmflAcYDE z>+t`R@PppDe3Oc(MWf9cZ`m~-u>S1d;i|C9nWz*yYB10&g%>~?~sf! z14;~7F#OZv-MKc+gF)&wX9SpQROZEKG?+SM;hs|%VZOJf1b&{0Hkj>a@Oy{BNQY*9 zrEo^9VShN+DdHenx(dnZabIaCy-s9mc{Z6dEK6Q^Lbv`vFT00}{$v9Qg-fMdbTIEfQ07prn1#3A0W5&ffRFV08a`nw? z7^^7BgmOGTzopQX^BtRhG1}nqm$cpuR=&iYflt_@w?$A0a>)W0q^Ul8QB^-LW@JOLI2kwP!Y`X&%@^NxF~xp8 zkIBMgbS)=jdI6vwQvxQa2G@W=A=sWR%Om>eag?r42`OklI)b97AL6BH0NjV% z9Al$WwJvzf!q4=-{qfa+InkAT#Kd0?t#@reF-^kCd_J zG5@JOeH-Rvwr99Ijs`%1AWSvVBU#0j*HBV7H!fn1;VnhOP)v~AbWPUS{m5U5zC^S> z9MI1@`pcvS-L|l<>q*rw4DU%_?k>f|2{H5+)b~&lcosz$HbUr^FNNN?ubg-s-2)i9 zjN7kcKv(EgRf+7u9?R~vf?PLdd8Q|3b*qn z6Z_u+k(;vMxXs^|&X*m^07&PrqA&T=k*v%#Zt#JNEsKrxd+gAwM#Vu|z=wGg`H{v= zqj>i;$4z{S`?jsKb7e$6VPDmz2uc-gRM^4%>1Zkg=I9wEIMn@tUoMb7ni&46Y_xH* zY9*atj)3iq#|HU}lzv-Pc-i4JG@8q#2O1Un_bF+g*cFm9r^A#rGh^UpY!r-G@2ocg zMXx}ZX3X#DH%RDyeNgKbQAv}d8dLPFa;GT!d$CnT`@dy*6@Vuo|B;m0_KC;DwhuZd zz@k-wEsE@?n*A^D_7g)8KlKsScgT)?$Wm5$YKd$v=l{m(v;it$|7 z2f2*Fg#Ib%j8=lwONMtq&+w59?zLhY)ls)1ZAzu1PkH?-y!l&Kr7<_&eSsQ!<{~{K z7v-twqt@4bVSH+gwLDf*Xsi{~K>Y1QF?KkM^tnLU*4JW!2jD+H)xoG=6n=P-j1kSH=$kfRIOc`Sx!=K(^Qb-_+cb28%NPfc zR{k3WmO3L}#=5lI6U7}35nhuQ0AK>6!n$(sPoOZkh`y^joQwIG*V74JNP!$y8`N=7>I zp7;!5AC!cME2n-H0?rGJoFs@e9U)XzgN$Hc=;>c?I((lJNKgaYs(-^&-6+sb^_1i6 zxNY>IlB!{6fW66_q+s|q`Y6GpK=irs$)Err$F_=>ik{@jQPoRqLen zY|4CW2$%#>y@hNcHvUhBZqFm5P*`<5n=~T(8f431KqNWvkJEOZ)xJv=g-P^;v`RWS z?kBnwKqAGuG>73hKKrDA&lknuYV;K9G7Gof8#{*AgpB|x5^B?TCRp1=-D3VRVZO@?kJ7A5=nSPk1{c)SewlyjUagOB;(rxmhhry>up}!`X?{MQ996Usv_t zT)_eDV`y2#Vx#x52EQu4A0;;`aP)+E?Z1M_`J<&s(WJJi7eLWCO;}v1s>dm(Sgp55;Uy zK?y6FqVeKzRF=c&qth-N%F-DB;P)>C+Gs>5EfpEMgj=PZBi^?Yd= z1JxD#^x5bsO~{xp%@C@qJVB?CGfDECbg`AzB^Vjov6`Fxe0)br#Pf%5_WwG8Y5!XO zC&V2`tYGCY(fba9>G^31=? zM>XBzBHUopcP^lZV|XU=06K#9X}<%Lfy6j`MFyi{5L{;jj13miz5*zh9aYf;y?Y3! zQAzAx)k(aG)LzKy$NJgpd)5}jz$ut7?ifRphO^1B@yy^{F+EaqW@ zsj#pzI}yS@V$n-N$9kq%CFjSYF}7v_7UER0VX+qU`npPfFDji1$w9m9uu4S=jO&ZU z25O=QZ{cWDJn(PWw=^>H6d6Y8b!*N;NeJ8!Xnh+JK8e`J-VHD{%|Fffx~tR0Usmmx z%Onxiv$2|z34ea&*Pk)5WcU)nE4It2WpYW%S(Pm)Gmy+B-wo91BQj6xiF zBAJMpFR1&G4V;-~mZ$9cbC10y-e{EmMw-L8bDDa5ym@yWuu7}%p+EuUz-lbZp!}oG z^$kqXch(;vzcqw!pHe%QrA{(&Wq8E$kks~cIO{Ohwa*xUk;)oo`0iT^mF7{#rw8(< zGX)SSA@P-UE#{BNuc z{HuNGGcI0pM#XmrZxchSx6fC5MEmV&wFgkSLiOV z&Y}sz^Vz)s3q8|*>dSm_!Yp+&_C)0Iut2o87P+sq+$rfl9lws^7fn%AIsd+Y;CpmJ zQ}(;B_#z+KeBu;lo{@wK-wg~Z{J>sVVx&y0km-E;=YDb52{Q1JF6@faa3!HY!)Qg` zA&>S6olbe1L~7aetV5MRoi*NxvLQ1vo~bKBLhVCTv&< zHtYIS4u^@DB(5j)d)Yt_eDxeI;KKv;L(0-X4e6@sZ&V%F&|LOl`1CJYY1+ zggHUzNKw@NIgqy;O7}fMTCW@}+ZoHh*XVoJwY&Z`6oU~dq5Ks4WsgnIYA7lDZ#q20 zv_zSeoc!5yNb`xOvkdK5AXI>hpewQP^<4;d{i>>-$Xq|1NwweT%G#3oC>pDq*#2Cd zxaj8}EKnt3{_oJbWV)188_s-{E>WuJ1?Rm-cg046=f~LyfxIfcB&i+;5pEn)R@>FU z4$I>nc6P`MY1cto{glb30OOB?h90%wHzd#YzGO$cY~M27fXUtAJ11aUwU9h=Cq$|( zA|Lv^-&omvuOdCMndZKHIAV>szLz!DxQx+lbEIVay8Fg)a?i3a8-x*bc9sFHzPml^ zgDRCFv?J=!cD=WABnW}`P-Czm-4s4|goO>2fOOcWb5dZWhAN}ro2nO}9|zxZ;XdcZ zG`$5v6GmI~qB`GwVX#h0S%IN%)qws6rmu&G6pVPx@tikYvn&=Lw;&?y)hHvn0gSNa_vW4xTz zd$eYlR$wV6T+fVCxI6nv-+~EWEV}8Ou@kW6r^^N*81lh-$cu`xXdQfaY~>^Q+22r-c?uAKckf$)v=%f?>Iy zn^A~%w+r{s9*ZeVbzaMDgj$^Km0|Vs<-?+qT8FzFtOw z#1S&f(Z#xb5{r*yEr@|;oL_GDjd@{2!h|&$hTL}KkuG2XGXt=^gr$~nA zddMZFUw=AWKl^Ba1$P#RJFPecVg*8RlReG`y?sx?H9Ot~g72P3;g^Uy1aJAW$T`!e z|B&#@=zOxMWK)GIFJ}iz;c5v{-QhpK@Ys2A_f>nYKO#(E)bnQjz`MHVF@%_1$noC` zq4x;3=|L8eMhhtsy}OQX&1?b5_6AKlIr-+u{c>1Y2C&Dn4nDcL!x`YsI;nFHlGM?64urfW zBAhpZWwZI;QjsW9N@`q6d*qIZCCjcmAYnkgtM9|ebJc^h&%C%g17>cdLmr>{8Da?97pWY>+eY5`8ylp zu0voW1uK72=#hU1aQTi|%J;maE~;1Qc&7bl?}5nMHn9Yt>!OSB5j?{7aJpy~dS+`A zd9;`&=v?|)xe6GJc!Se#I+#!`q4Kt_r2N#<6?cB^Va+k#zBOn*R+?M#exsU>!h^G^ zG`;QOf!wWkXQa$Jw4RTeBY^@Ux60>-@`JHITf4S_fb++8FiK$n>i^PRz@8f4b!YP0^-5(&G(TYMB{+Oa5OVVN*u zsLqVl&GD~L+mgXVns6GSZ@Ir*PaM44{Kac@Yy!;f_yB92nq$C+WJAI}5z;FEL2+B9ObV@M|D z2n2BUJw;NtKMvBv-6mCT+D4Ug8=I-wn^k}~m&B`Tg5UU}_bNAYUb4d+xh@nB_YU&f0IU%nvcbWuget?D?NWlngH zV8uluKTV?1wepNie5}dBT2PJ&CjTDehFwu8lH4Qm#3A% zY_+R8=Qz!G=^-}T?wf!g->@>|L2_~~2^Rhjv~;54vyS&-vpKqw`0~3nAJ#+hb+&_X zMu>*r|DZ)&ZmqrC4D#9f5AvG{2<*y72OH{Iq~bu^%Avhf)K|g!^T$3+IK!!&!L|E*95dHGjW2jn`dXW1PO&@>pZ3L<~ z|HC($Upy_fbx&f~?k!tMi{+i$yU4OKz)#S)8rk!Wo z=B_pN{C!Vu$h~#B60(lQsUd`e^tD`e)=+P%YIoq>_^9Y5kG0Qa6-7@SCTSb0FJUp8 z1AbQTTz4T!U`T-`Dl-t0JKhDzz}XHFC}cez{*XNHzUE_oyX;y*xp@9vcYRYl_qOg^ zUD%)XlXH|bk51;!#9yy}TPo8BJcP8$tmBy%ty3>b2~nB86ljY{>VCqDP$Igi(=nZN zdl<^Q&SjGK z)w4%^ODC(3RJDWKV6B|j8qO_cw%h+y|4U#I=vR*^Pd|opD@z32vK@Y#<*=PuUG+F( zFZtS3P3e7wuiNM|-ijikXI~BbvUH|IU2KJ$F`xXD00AU?x@#01DD6Tx6=>QlW_@FUh=)R72J$A7NbA9}9RNGfe_BsqX%bL>pXPuQxbvQvWXNGT17 z6?+AI_p_7owN;>m^mF`H=GLGra_O~W-+XQ^c^l;8h=DHZy=D}HpVW<9EJBHj+;WdbgJaGG0%Fq@e;5@;Q4_sYQvbZ#qfZrb+0! zegTF}O67a9c#LZ-svy=Ov{_13$nw@33X=}ZS4uTg;%Hj-L-_t#VV0Eh6_9US8Iusq z&)_YC4&h@ls~0CkthgL4EZr>G8zPke*BbcIFQSyvzmal$UBTlvLM^|Z!$hmqPNSA- z`ZeCHV2j7gOdz0oX8_6G{Yp{qOygtGWS~ejc|?apu|^3Xe`eQjPmuEB(0d~z*i%xy z&k^{~`8CC-wGLMYgnol|j@022%AEVl6jVB?=WC$oGa6}F^!6mF6KD2n)A6ftq1e|P z;f$;(OsMG$F#Nqgw2IP9%t;(F%kufWDBd^PryLY-XcUrb{_TiriL)u%7ibvLj zIM!q5ZGZ6g`E&xkbt?3+w? zRRwZEqg+b&WiGczrMW*>3Ht}rB{9z5PMViS`CQJsPx&d8^LI7%Ub+99n=L)KpRF+_ zuXy^@r&zOVdy+V3&_Fn_mqhMG0R6bk5P!UkrMu{Y;jJ(1UKueaOo>Rmd5jdCy_(dx z=_Spo)@ww3=CoHiVR5PEK^{hqoLQ5IupI9-cZKn-#b*=*=Ca)> zCwuqlX#p0{f?tg!-{AEUo|jSTYh8#|;E~z~ZeAGw73vI$`9+tDEl(xIw`lDVCE^Il z!%iJ~L3nXAeZc2Eic+Fe)N0$N&bY=_f~$;JtExbq_c3SiP>=-%AbsV>ndGblMDK%zG{A&n*fgU=X>s};7y13po%8H^suK_(I8IMLf?fyBG z%OPCR*%8gLE`ddR&~3?-Y3`b)D+pO~NC$0w2RUm21;2*&hE)snIpFy2sLa)CUjniz zYaOs@Q@+~4SHA6my;9cTxCiu^8uIc3Bf<|)MpFf24qZwkf0lHjtY*@F+ZOP7jX}sOj;V0x>?t;WG zc2xeMMrv?<=4YnzmnPoWCgAOH1z|u!nz6=4oBm5z) zM@`oiP02qR_D*-M@CPz3CjRN>QSGdN1@ik?rz9yNyU=HZJlv4v-SBU&srF|-o)b07 zhuHn;lLS<|&|{o>1HmRL_h z?LoTzC5t~CvojLe^hscGPRe?5rvxX2Zx#^{deAumn+shNX%#4-9URUo z*ODalmlc#El?h%bKc1dBq(!;vSZVNlptzB&7OMZw^#J-@@c^*I$<_g7c>IYX=bxnt z?r_#ZMtYhKx|FC{I(t8^Y`Fc1m|eHz_1+F>*J!hHeLPIHKMvg0bokcRL)CbMv*g>L z1Mo>V*R@d%aJ)#kZmy3xX`0J$V?s-2aY$>uhq{;g_S-?w6005$=E!vCD({{5bef!Y zj*&47)w*Z()SU6GjsR@Q%)|!SA}L7^jk7 z7gC+QJ|TdI6%&C6$J(~>cko=(9$Var77VxZl|nwe_{tXaPBm`CSZ?LZ20PH6m%J39 zuJ*!^g=yi(dNh^;V!9y>d=aKwhAF=rP=`77p@oXXqxGpcQq$xqA+y}9`Tc>C_Nv>5 zs?TIigcUupqcvOVS+_b>vGcMaPfs(FeP1C^ii>OekxF&+;vrKXXt$y)ph8R^Ct(xJ{e_C%& z44XgaxKiKqoRzO!B$4*Uh1%ap+fpL-qh@i#%CS9sAlOuJp3n8g8r}F@JcVN)7N~|w6X+fB4O0%uj3v!!xs(zcUU-J{T4ibx} z$F;>^>hm#7RoF?|q(KC(I3r#_7=aHP?QCAWIMS-8770Vj{xps%(52&ZDO@+1I=b9u zfNZP#dEfnacR1D?#Rxiw}rfv+}P^;px6^#x48sON!B< z4GZRP^|ft?4JJ1b;?ZWhu)+qPLO6sb;kfx|6zM`8bsT55C)EFiYZ11TP&8Yugub^b zxb81>gEZQo?+J^?b7qljy9Kcw=^goYm+{>v8{!$V;COw{NFIt))~u__H|Lc_Ggr1OqjddUOen}mCngC+ zU=Uosi}M;^BHA9Mybi(mD4D;q0DZr};0tB!(Z;_nB#ngj6d}yxBV95fOjsF0k>SMA ztA`!A*}3?X5Ye#2*!Li6mVX!1+N zv&$V;_fjCzY_O*t8$Wc__CqPjO)1&V2=0kiH(G~jJ^)vc7iUX2X|QI1-P@b(Hu*?b z3^1llzI*yANt>MtD z51RU3ufERbGd|T_syk+_-fCO5lT^=`B<**t06u zF}kR$B=jG(af(fp-Zp#b9Y{$Nov>D#_9U&V)?iz|I{SJR%yhnh!j;^0bKL;c!gA4o zEEz|Hr%K%wZPQ5+j)$qDIMSq0p1LoMK%#gYOsm919UTm&9t>-d5r3yG6Z)!9k~l;83d9kct&_^)Jd<$Nz#OpMnBnjm-a;QJAl@zFzFbs4z>iU)`4 zqYnp;uB4@y>Cy=HI!r=v)zD&XD1l2B*GZ z)sNJU_Ist25;Far{rH$$1t}~HoI>CH(^Eb<6nOTkwE(4Go(sYN{W3;we{$sgXFTZe z-|#cYuUMtVd8}T@^+qEknlyc!8g4|lgg;0wnfXY7L)ypEgFoJ9m4IxRO; zvn?P^{#T`U&C`PIumwKY9_GzDs{2wFHVqPOKMB_e1z!}^HD-$@i!rX>H7?xry&3iH z1_`Yu{v7A=SoiHH3y;uCl-S)Wx&hv{ zFR-7o>9huWCDnt_ftKkLQPpE&btgq2s#=-akvRsd)MMtaCv##S5zi{v5ld%LE;Q)- zm=%|HRuFtncQhI0m^gj#tQ{gf1ku+CJjs-&;p3;EWaI+Lb%3==^?pXjp&nPhmbm(W zCyhBXnbr$N|K{~Patf?@JPZDkEkoayJ|cw)0cjn!$-~ut%PtQ$Ex)zizE}R)8e;2X zJrU>r?w8Z~`Nj7NiF3kip_<>In4^414-;-8r&Fk8Tyx)inMHWP7d~s92&0M7q*Nwis2?-c@Y66TJiX*jd~cI}+) zmCV6$vkpOQ!FO$GsN3qx(_nv(?f|!`I!)a~`9%DuyNN}&)1P<$G~Esdm~_rgDa$C( z=afb5YD1~Lmy0E64yIZ{XUb}DL+=^gbl09any0c(c9w@V-Bl-U8=p9PX{&Fj5S>Py zMk;H2(cUMceowYgp+~3rW7JT24n3SH{D8(`8JCCE@Mr>@ZMuZd{PORl$l}QqHxdoN zydN0f@a%_~AOF-nbJ*-TaOiZAS(E01{6;Qg+g*=mu>yxFS}H9Gt=F2H2;G{m8mXR0 zIdvZS%8=5q#BK4rcd&(IH|3g$cVoR9*@A=3t_SOzoE2^NBuSd(Qp*-et`<{d_lsRQWyj#AcuZr@K-AD&X~r_mC@|*I-N0^3alNEthfsLV(t-=XO9#8u?aq z3sq&y!6)8nawHCw$3$nS@<*_h;-GU3tksS{B}_Sh>5;g^zO5Fgaj zBlID$;JAev{6yvm=Ka{&)|o`RrhM>PYT2kk%aSm4l>e8>!1D{f{n*Dct3-K@PxL5D z))i3GlMX~N#OxY+X|`nAZ+l$DB=e5@!ttrv;a12r-13=O$wl_eS|Bqkv!Fn0n#&JWk40A9=fx-}oHFC5IJeR<@em zd^uD@dht1*{f3Eqw3lGes{f{ba`$J$xpPuqRTW3b%1TAs0+qK?XOI8nlPmspT zmK>28`7=K=ZM*muAANAXVcVH`8AuRx=&PhD{1lvHJMC^Kqq&|f4*W^?eRG|0 zqQ|pnAcreH5XX3oG#IKI13q>=tT;xb%QhE5mOs0ItkW8`jPCZ<5Ek|fHI2^r=6>iy zT;YX$CBD%SULap4@#B_TDm)}lEt)Z)FfjAMOVH+KC*W~vtPQO|mXj6TeC&FqLiXx_ z<|@BT+T~D#9KId@`-xo&yk@=B4dcRY6}=w*VApj^JN!u)gjorQuuH_x^fMdLZIBVP z=V~FBlv`C*iPK`|oQY`@t>jDsdADftpl>>5FCE0)6E zE*Qaa25tJcL;An}l9*4>VKD9#?rq^63Qcl)O94Es@PgxdHvtn~|L7!A?=N{JlnnI| zB`Zf>@ct0fp|P&atfMg?Ho`Ja{JOvc9h%U7a<$tb25#jXzR<@H~zZ;53oFz>$uH+hsxm> ziEQ`3Z0s?iN|!ykat+@?4#FKoTq(PlqugQQ=s2_Gt_J%p=M(g9V8~}j!G^8WB|==_ei{~? z*k`WY`6}a+z0GE>_D)<@i%CKwI?R3RO~;Khw}v9P1^I9v4oFJL+}aO4vtnyp^m!Dv}@o$F}1 zjx?MPnU)F$1_yi2KK&t|0K-s__aHaNDd!CyK%hbV*h%E&AMKrD-G!ksG0R?9{Blxw$`@1{G_j9ZD z8jw*N>v(d?1B5_ZE0Q)n1@|~$fu{$l0zNd95fK}0rt4o{7;4r=y6RAJ1j{-9BcyVawQ^Q*XT6EmpGdV^+ndq>gGC=U|g@8PrX?dE0jHZqKitcXWhpm1CaCB*{xV_xZqJ^(*EhgOth~OIuuZmUmtyK+YBD4?c_!}Hv zn*WcF7A>a7<}d%SF`f!O(MZg%#_deYJV{=`Lshrc(ep5L9OA6xRaGh~PUXLi3m5#B zcUaE3>m?}sk=`eI^fr~0q!i4-;xrPGmeHTOaQa)<6-2<2t^4$saScXT0~eyo^9=je z7>rPH>FIzhjgevv9AC5{fo*|y*}L}oE#Nc{ivW$6h4dT5+}?luDSR52jt?)u>3)nC zf2HgyR{0m`_cI&!1M_E_x{1Zy7^vv!WEiD6GJJe!g3l+*DQWK?Z45JFd{Y~`*_%Tt7P#TiG5(lxI{BJ zwx88+*FC(JF~7Q#XtBAbz>!`Enr)M(c8ht(_ahfqT;bejiJPk}Tp^HLg8@=TcV1lw zQrq9m-}59F`x&$u>HJ4XVm~A+Xhpi>Vm@0^+j^p$ds9?Jr({S+9AIfp2u?RU5jH#- zlo2-Y4AJ-cL72*mWh7Ze2Qg-=o0@RF5N(z{J;Ke({edbC`q*3g8uFz}*Ex2!vN7Nf zU(bYUBv#x+PXA)l?_lkcMAEV*Z>=^ztm6}+yuO-Z=myAO(>B|5y}T{@P_ zo5yaddFJX3`pRb-UoPdJK+$xOaLwg^G5HU_>F`LApTvVq#` zu!78!W@9SfeS;;W(-D5fAM<`bv7ry*5byJ6KezqPYn#`zDST46V=aId+mL92l(=@% z->27^l(quPN=mfw#15j%E!-7UnB|c;KqBENL{oM7&8SU;!Jig<{H2$sTDoE&)q7FUYGgK33sKP33SMVN*N$=^|MaO*iz{-W=yWsec>tj zgYoXdw4Z=5!s2P-+xY`|*E1pp$durr@GU*Fbmna_@UiFbIfD;UCM7_eUbPuX#LDKd zNXgKHdAr+>Zk!}stzHK15i5@!K@>f=4MVAh!TZwh8x`fVu(r_WZjl-w@5aQnH!^u&g%;5q(<~hX zmyz+d8c>6t9X!fa-qQH0lkPz3GUmIAU<)0xiBt&{(TK^hBU^LGo7*m~FRx=$WquE= z5WrGePQmkDdft@M!9M|ut^N5D13_h8S!A7yFqX0j%z zsTA;X0u@QpNE%vnlWpB(wqB`jx;~uj^GoML52+MAQOB9q)SI0Ys<+0;Lf5Lai`aL# z3MI&%|l#Se~qb?Iw{7b)#9Gk-HtCM&&VDD7{g_sa*-EwH{k`M9ClDC@T3 ziJmHIjdQtbcGJabuRX-Nm~G+;(o|;s)?}W6t7>*-(Q@!$;MTq7;%7J{( zX@BS|KUnDflZCa-ip(<*2gDBSwp8vqt^2vQ+R0czcd7Av|F^+YXfO(ya98TPzCO57O{f zVOwX@HtA1u$&VrzUZwKfw-at$eW`VjVoHqv-tRFe=6I6CmiD81b?9jep*7!9$NTHL zZV;&yZg#h6@`?4L&sBZ`NX;CBPJP9D1wyx(-Z^k&qw9cRgs5p6Q8jKRr3M%`G+mRY z_$GPG`Qif2_^+CJ$#@FwaU~&`q0L%~VX}KjU11y)L76l_M&ck1UAPC<)eCE(a9Hkd z_d|~#9uf8k+SqP_G*6^?H6c=cAvDok_x7~)CpzVKkkPf7V=mGSRbkjJ z@ShNqJbJ^=(O&jT)5B+0v1x}vW;-26{qvmx?-wC?=>SEX3n!*~yv*K+F75T4UZsc> z^X{0r&(sI~)+Z8^;^oU%3>>~6azwUwh`s{xsHxpa~hfV%`951qh8>Eq+2NzhyDGw5+SM8;ki-`=)ho^lZV z+9e9SJZDOc>Y840U+R?s3b`Eo_|L5EdfzEgKk{F{NF$rPM5*UN6as0Zha)`gLRdoJ z{#)dp)Wu(KqgnnGdW7TRbbLhdRuJ#IPqZ%TGn0mWhDV=W>pdSe>m=4>p&zAgmr&h1 zSgP6f)5~RSWT+upPYSu#(C)&FC~ZRPU0|Ix7C%y$9fli)mxV^js9vmIpB#$g7@;5(8F*|_9*tMA28U$h zsg_rDSE5{=;2P7`x;RXv`t^vu=&hiWCJHoGyJL!uBiP>;z{F3A-M? z?oH5ZHAWLi>->D{yIOwYTyj#iRL-})>IbKKuE zc4nJ85D-_@>z7waT;0ZksLUFc=Of7N52D|n1xuxrC(UFL#@Im!v=x7M&A8wy z6%~Hf+dp@`zbORnQ6}&(ncg_uwO}e7LwNxEMk0AUgYem4boQN95l`_DSc!Kh*D6Li4Kzr{3>e*97WJw#q!t0K=y~&n;zO2}~KJog=J#CLNcH zP2TQPk;{_9bTIx!7!k!vX2bwRe%^)n7t7Qpc4abz>oD(sxA&_2<#V&Il47q+-4v?| zA`gxz_>%fs!eICIhRA^BO!YJTs|6c*( zOJ_crI|Uw@60`R`%wB6Qp}Lj}gk7PEo3pFdDn0Up+PD}1T%ZufL=c(VtaiadG%}RY zup{K0!3+37#hqvT_y2hTJkRLxKs(AMrA-+__xB9^y03XGbhUFB02;1Us--v8WWvj-%yk^m&;r_>w=%$)-RQ}WH(+5vGu9B zS$|T(<5hh?nxoH-3O||b{f5BS7(aJGGr@gE1V)V00pmzz^T`X-+n ze(P!gY?ECZ7hnHCslVP~FVzj5RgP?Q@7QviVs~%0_=i-ERMSUWY*#iuW<1hLE4pI2 zxzorKUHue~TwkZyXILa3XKv5E+&WTtcPyv{zM~SQnew8U)nsL)OR}# z?eSUsn}W;^*Yq)EnzVVlxpDO*uXzlWA1+5JBvJ_9=Wdnxtq2)BZ)q9bA0Accckbul zUP%E#j*p&Q3>b`)s)5XASDh68|H4bhKEEtE2Qw#dGWXm1xN$$Ccubdh`=`&|p-HQ| zIZt&5Hjr&Nt9cPhUeDmPnY>gyPxv~02g{eGM)a?`zOc}yIC&Xuy{wAHDl22$LtY`{JQAcGB%SIwnv(+fa&nz`MsoV7?X_fw)j zI{BO>E$|7~n655ppY3{+@VH&^VGSB)!(u?TI+r&o?$7K`oQ-%O=f4uwj~vxY<)SnB z1E{#79Nj;0;|&r2L)TkIMfJX6qjZ;am(tzMNO!7qinO#yGc*zdC|%OsN{bBLB_Q3M z0z<>ZJHP*X&RXZo`MPI6v-jHTdG5OI>%LBmN!l%mSpvD5y=*FPdx!U0S1-aqIjysY zBmMTdb>fS3A;@+G8l3O!TUZsa{U;uQaD5Lf|6*!0g~dM3vw{yW-TcxEgV@M_3z_!- zC`Q|PSLveAJ>c|{)ODJj?V^KpcOcd1pW%Eob8+;dj(O80c?gLZ$}^?Y*9CJB5+h&JU3k$4F&-q`TWQCKm6N8hW zFwl=5x6xt|V2dzA62*Io5{{8(8WArYYr5r+k-j&snAo2DANuSxA|-~KFwXCfe|}v5 zHea+ld|fxm%v{s&cVj}2rjg=r|KJBAZoDMJ)n-!3&P3J!rNS6rkupS&r`z)ut_cF- zaX~#v<{=uax9W(}mWL{Zjd{-PI+#VlUsAtuw5r&4B;2T?i zW)VLDM~IJtZToi(e?OM#1kll+JM&Srca%xZqdIK&4?NM;awPBKE#5ImZ>j4R(I@Bn zZA75zX!?pT6{dFGwo#Rm<~qq|e|3nor4Ius)6_G>l*#7Q6(@TuC}9H1|+VqW;)K>d3o~ zN6|TFZT2ZFj+Rg^jrKuDCDjrp_o!=so~5>Z*eM^ zgl1QLncfpRGT7d7G?;1quD{Ohwc0xp^|wXqH4&fdf1eoAyPaU=iqDZ}ENWgG6%YfS zlpDGaCZ%NK2WZ{Vwy_$XEJMUB&@C`a>PUiv<-F7KLKgxW@vqZct8`Aq8^zF{-WNcNgyq3am776Y1|C`6v`$Ej>%^E2!$MgBmOpB1?=#^xL z2JE+mAf?<|(|T|m+WS@24N*8LmFkObM7qHHQm&}aUnifXTqhFYvryMh_74`#Fg3e5 zrhw?-lMSXznj}g%`YyK1G<;Ag+*yzQ_cBEUDc)Nh{Rr`Lq!gEsyvB`Qwe>8_&u$JF zl4(8shi;#Zgn%a2-HxA%)@pg?MT9Umt4;}qf;(hZif`}uHEP8pC;D8pnl+wVXod$+ zTFA%fmE`mu>DbAkus7F3U#tsE+A=bFGNnQFM6nQK=%(}I9E~Z-cP+xgxqSvUF~`c9 z^lwblE(;8iIBtR|LAB`0PXKnBQ6l+D?H_Ndd4u&i^5})j8M{Ay6Nl zUqy=}>KSnMC$|4iiDTV=j`;{^^ZFW>UME{})V<3Tf5^t~VjVf9BWu9u6|sFW+g`EUHCZ-|i)*>h^%-&xP6`H6MHePL=`NPb>HPBT7Wy7-jqntQdtL z<{gmE+;{DsqnL-4m!&z2^T=uVzli~a=P`c!DUqQ(vFG3l?OsBQGLq6nItiomkHYm8@<%gaMr4Iz63OQ7w*zQLCbt-DR?0F!snrVy4=<}k386-5$*RYGD zJD7&&WK4d{$FIiV)%dYv(4qaif6~T-FpcriM!=?0J7KxvlF{alD ziv&**VPa!1`7d_HO0l6u5{MhQ9g^h|KfmCC>_}krDCUpOLjWAz zhh(%X$_K=mk3hB8lJi_e6VBRfb2WX)QTUJDyVdQ5@1|zJUsfG#QBrUAyJMLm>oOZ| z%nn#_q{U5ua#C0A@+ACq(XEqjq*(#v=z1@^8Q79#;RNnPv{gf< zs*~+L(++24x)QGlkHi|?)dO$<5*aFP=4lmc&sn~l`s~?iiac#6Ftuis!)$d_ly_OA zuf{5X$!^hJBW;YPn$QXeVN1+X6vAE|Xd-?5r^UH!&m?fXLkH46V3vE9C=Ec~%VLX*4){VOS;Lt*L=9VVk&_z#mLZQ>o582I6` zCdmQ6jeS%DJ2}&+QY4|2B#o@fNYlEtinU(^SE?Yjj%vv14<0C*7Yoml+$J5e+-UH` zR*6C8pIOJksX+hPvA)VQpqsORc+=qH!F4#CfkVGd<-rL-WFv+8mBTx5X^r_1dHFoN zf0Z@38F;87<3Ua*CHK)i<@)h8;;v`g?S=db__{b>A|8~-C@wMb7+eSp+PZATVLIOo z_%lT}sH#hxOScy+KLeKY^f!-6Mv5Cd9fID!tcaZ?^pK}^3vxm^O25it7=$D7xnn6uyWBfIqNimv^wip`sJ6-_}$aVCg@&A6yd znU9bX_WZM9va81p92`cVg&q&*EtLCK@hcv__nBxw%#Uo*Ah*hoO{_%=z1~H?jmFvf zb#6l=GaSOW4ne`BpWB+NS!I$;+eMc);*hK`SKy;PxT~~GJ6BFmB8i;(o3R;1X0;`# ziHvm;zqmkOhnNtvLv_7_C_*T_vc;j$rMTo#DG}veC;Pg~-Y8)4VH?)t}F z+ns?7LRU>@vF~n-@;gYnM9~?gHb|_M-_c;SzU#S&!X@(>bc<__6(7u#dKYM(c*jIC zM}LS|eNECMK9%(bmxU(rqZ816H<=WNc_qTAT|ApG-LF2Tais|57Dc6*pw&_a}#AzWBB*m2A>j&-F_M|FdE4!=^idnM#RB$xe<=EY~KM; zI`ox3Ms!n2X1nsv5{a?!yrR)kX|C@IT zzUb3oTz#nUZ2hwT3gEw_&U@6DHK4p^e+u&LvuFDR!K^(P{CIrnFLn@9Y?up+1JFVzgGOy3?O$7P^FMP)&xmjB;n zgm2~VUfRe&w~~$Dd6Si5alGic#9Y$m$3?8td=ig9$Hw=XAgq9Nz?l*DqR7BE&Pkq! z5#ABUF6Naw`4}tC`QRy3{gcKC1k+PsILt?jdUL6k?l`69@(ay2P=wCVQ$$ZuUcL)vfp*1OE-WNJJ`DG(Ns&&moR?B=om9s-4rzunhdi`%M1{L;QlR zUF?wZ@e%9P6(~Av!0CQt>h8msUPKb^9Io^U55w%UXo{Mhe-Pty3hZx0AIsKeTR|={ zyR*V>)hEyWVvkevR;6c?XWZi)^dvFN4IDT33|tO$>S$HI<0nVk-Q-#RXkpH%hXMeY zM*MB_s;XJ^C2o|etdrbKXolX?8#DCpRaMGncOk>_`{xlWQuQ0%umP%u4u?Pps@SJA zHGTOX3##8zzv#`1MU(S|QSD@iDnTKEcGlg+L!-tf~Xr_pk6Dl+L2ZF5SJv-|9Oa!I2u5LdneT|eaJ=GgulbHB@^~CjLY(-3=gmz)HCJ8QL zZYlVr3QeKjf^)%N-WgFsef*1r7KyP4zGYg+3K=t+`rmD23mgVr0hsG;`zH#Gd`!nRbq2_X0)7+Bo zC2Fee9fu}Gr#PiU7H^qOlLruS2sv2UBgZ6|@Hc$R-x_1Gff$>}2t@k3+G1GK>9yi` zike5@TozAa(DjiGE#ecY`YtdBagv7+it&+b+mGQFj^L2oX;I5}Cmv1w^e6oOEskcI zKz0O*|47ow-cFQzJ^Q8Lyv5C9G(Iz~w#kQD5}FkEh{|S7lF;KBdWwy*{)7bC>Q6+B zSafVlQR!RAA*8F9Mm%zvaE#_y_ifJ*sIL-*4q#;~HHeJ`VM zdc$4>yv464MNJ=P#CJ~_dG{XL;`Vh)w*`WzkdYZ&8$D0h(er}c(v z>~lZ`h;J&)2nvBy(d5btJ)sz~m9%2U(Aq<4<$x%dgPNAVc0|_%F6(-v!<32C1R zWx8b_-@5;-3^10D6<3cIp>eE?QHB1`z8$W6v)Fz!?6sP-5zOAubyc5)7$W*i-|V<} zZcj!3d%H;$66Z+K+3shc$-Xik)AfRBCD@nkOlFufYN-5KeMjprbR5U9CT+d&KEp6@-)F; zjU%==RB~oCjN#Up12rK%>}}D?Y`BH5-XhHs<sYYaA;pho-e?-Y4_=qY~*X5noF?qV=D>Gzw= zQ1U=MCmLk7!{0n)%@AqhL#V=W4oia`o1nC2$3-AXx5Ilml40qG8Szw6`ziEBI#)!X*=9ReZKSX^9fP1v13#7B1!Q?^#kl?+P-C zrxzsi-7bcANpgO6%(~%EfO%09=U)DIaW^(XZ(pIy|Hgq9gDp@5?Exsy?ohpGFFS@Y z)Cd?v#6POiBhOLK>psBOl$Y}yx86f&Y>};TtetBmd*=s@WqAnE;V57s%QjeO!8#7E z9+074&^e`53MxA?E^+os*I*}YoDG?KtbslZHN09D zMI7OYOoCPMExvOqs0cv2LkuYyJ)D01Ov zSXgTqXg^Ws9HtCg{MTL-ch%ioJY?~nQ*T%S`ViuumrQKdl-s=CC?x`_g5S$*7mz#Z z6$5s{f12(9+|$+)FGl)_-3IhSYJsy4N-kQvdbB{JsjJKb%y@H31${^rS=!}ifh6g! z6^6ErPwPaB=ocTbel^8u?k4l`%{}BE6i6RhQ_G+yFR>=QN$}E6Qi?o%GC06{@j=1; z=&ULW6+~+P`un+ezZk5jdF#uVvRQiHZe7c5N5{9VNzG9UBUYF;7PD(r#&nI^@_?{K zvPqOk&|2DK&5vMdl$=y>edWm4!Tk=j%9z8y92}&KbyQZKhV7|xQeg&70o$e#TVv7c z`@=Vxw=e%-YO_IkvIRVD)a(*!sW<6G4vc1xyjdb@r#V4H+Nh4I``HO{ABD8B3cnz5 z|7IbRe6aJDbyeJT>Q8*yIVR`18V<*|nH{7sHn`R<@goZe>kc>>0I`Bu#nrGXsO??J>%IwRF z35QY{9^=JI z#44$C)!Gfp?8g*Ap8-=6xvAq9r!eif*k=LR<~zM~IOLlm*Fg5~gCmrTNjvGUgfCy4 zb>ygrgs!?>){`^2U97!g;Rh@|pEDq0UU~X+7Wi@losR_m2_!F5U(55JhOed6E{Ygq zomzBgD1HiUykC`p6KZT->QiQ<=XAVx@;zk7{qxwn;c@XG@hZGu$<)NiT<>6zRYAzrts?B!wHVcwt`uok4zP zCi*3eE4`9}JPq|*=o=}9WGBU?dUqn;k5r64onNs8u3a_Tw_PL7@6(Oyt>YghS^&4e zVj$fsNrKYH3fn~l&+|k7h2G%OMX;nzO4rShjG%~35Ti@FCt&{fY0_-aA;ZlA-7mBa zz0cy#WminsAvp*^?{Lz_%c3>KMXc75B1blJ#1=t(rU9pq(={PShwk+2*C|pIrtz}3 zj`--Rl^{}S^^eN@Oqu+4ictohsMP$5w~_?mxXGGcjk`4Nosxe@kmRXx$9<$`JY2Gk z5=(;g@<@m0V^=)>&k_l|!_W+sXFk_?9Ul5jqt*N|wfmye+6qf1)sT-&-vU>M>V@{C9s&LUhDHYv*Y@Yn_2}`Eu22uF(W#$lIt0)cvEH&7Xo&&Fy z9{je=m_uPW@uo^do!Kd$%u_N|vyt}YM4!c^?u3zTS%i7(g|nT8DRO5xV<=S@m7cY;0E$OPy_K01=cZi0Ngv{L1 z_LCy`ZXf!EDk|A9*wC>@kw2%@ccva zeStI78g&S${$d$Vc7b}al=GFca~4cl|72oT_`9r{U40(OTc;wA7?FXU$VgNIbZoMW z+ZhozPj8Xhn9YX(;t4JNG0En!OP=QPVdCxeMH%aOfdV5|Ezg^B4D{H|ESB7^SenN) zN_!Sh>{vZg%*r_^biY4vIkh(+6BzX6%uUEE82L+V6570TC>)}(Psg~EbWMHjLHzZ% z2z?+Fb7|{x#vn|3a@ty6K7a9}c zGFtAN7P}wh&<2<36@eGP=lILXc6-VHV9?f7fedS?K$bNhX7r%2_dMrv$OU>+FD#)-))jt-iypBRb^ z467HGbB`%D=wGVi4e=)QAyx)eaUXmc7_P0UMKUIlN1u4agY>}|4Ly6JrZ!Dz6O@4z z} z!conmSf>MZ$zwuhudI`iIMs_5)G;-6L}>A)3=6M6_mpU;vLF1lCSzIxe(Fykgw-e9 zVvO;UNzSaq;|X^Pn1ccSr?8QhBAh3&ak<|4uBmyz#sK|1jdCscn5?Y!@a`8{pO#j9 zyw*M$`U^!;IH(YPaL-UZs!H|?bpn3uY?j#{gV!&&f%5+8F7c7Ey{fpPnw1I*0Z8kir#4i+CcE~>OZ0_Y<^MA9F`!&Aa zW+mWrIQfQ!(J|>UK?uRC`0ceMy)x^r%k4r6g*|!u6&v!M+8dd99ZN%o%&S7(Uq4){lS_=rsdp8-o#wsQuD< z4NtEk5)CY{p8AN}(+y46(Y&w+X!{yG4kiWV_R1J1BNa(00#fF1$7L^*RXFeGKtOg8 zdrD6)g#_Tw!--VdN~4CAda-6rv-f2Q{&^S9NkCtDgLk6xn@psClPVcV!$h#dw7Dn8 z7O(akzg-Y(xTwt&;r3*szpe@{$t2+V%EJ`*2uCrKDgLaEGDVM~`QQh|v$_}83=IQ) zJII=m@jKz%ud|(Kp%n~VNUH7fhY}O;Ag(EfC@Cmu46T-E)&8o=?lu#|)i1bj_&7yndx1+2Q?9+d1q!cfB>{PXOxA{J;V2kw~lo}zTVV4A` zf|kbuH-5a817kHCvjan~8oW0!Gc}XH%Z$s-aAu46)#IJ|rx&}3vp;D;E3WZWS-Y+m znc@(+jEJ8yr>?N&M)^ltvbMA zqNV#DndOkdB-mk#60jn^CHJGK>Dr$h5}Xi~DT2;cEu_hmh@9wMF9AQHbX+_i)xIj& zl!Xq|#8lwJXYH<>_8vD+UI3BwV}vk1_^Sw-wQ$AsIJm^ZJCfCUFR>?c;`kbJ!I`;M zI6@`$@{-NxsS7GI#`eEA!U-Pz-%{_<|DUCCT?dhm{;xcaBueZjmNX1e&_B@it7=Fg zx76!YYnhBFL<^GRp8~Y|FTJrT#N7dRmYc1#y7NuL_TNb%-aDmzLJFlw02^0PIZXsgOe#@8PcY!$24C*Hyfqa3h7d zI~VAiVfV@;W*u7|ykv@8t{%T;1I6Ej`8Hh5C?Z(;Q38dnnW!d`z^jT&t9`|O=uN)_ zBSh4SJSq#F3+2LtzR@#kyPWPX9qL-4?MvGk3dcLov+~mgI#;tXbI6T`XwdVY9Mw?D zFNk_PsTZ`2Dp-!5QKhCMeLsJ8*{K_wF(}ZMY#V&=#2WT~C!f;L+(QTnu0%Nzr~J%` zXSVeuJk(^coa`!VW$N=ouv1hvo?vPVN8@Kow4-D8a7ul%AG|a_lm@^s5QiqTexeoo*Rj5) zL&)GC@Z-}))TLg_-olJS;A4x#-aVBCD;+#oDd6(Ex&18>!-uIZ&3CrCH`s6u=pj~d z=l}LSJ7RJ1N#quIf*4vuT$by3l`PDfxi8dxnTay#aUJRSxxgr0$3Tb zRI$(x7r8>j4!s@JZ8s>v%VA?1 z;X7^+IUqEBji~h&#gk zjHwEUM?^pyP$ZG~`dtzA<^$D|%JqA_&JuFGI7kl5K-pO3PO%EMil|-5}L7nhd%>)S4+7zx(0pho|XzO1NNh`x{}ld{wXljsPK!^#A!@wevM}=lwE5 zVlZC#h^jEvICsquqhOym<*SZbYD_fq!9Wtjt&a*gwb}B?51ZJz-3~!OTb--jPR&h0* z@ya%2`!ng2_b&D^+kg+VbNel}TN;L4y+@xauDrbI<2@$7j^`#n#zplnoh=gbu09%){wVgs39kY z9uamM%<(Hah4FBNHXYP+Q|4K}djQtLa!f?xPH6Ee(}TWTC7F@n%y7$u74#wWGNz#^ zRM?}V?HTD3=ykKLEzJ>bII5RUiB#{Mzb2T-@4K(Gz1|qWP4zTaPH%SI)YVK#_@lQL zl~q2?J2U)Q?*7{an2raVsl8zhxOz*gmvunm>AWs0{9rWz39$tJfZ>r)rJU;SOAx?P zlR%_W;nSJUJ~kL;9+ar4Uo&18X!RlM!Z`sY{~7Ynu5yS546BEO_c)uN#`^zb%U3+=PeS2(yVE-jU{V;zlSlk6BEAn{ z0j8pAP9wG{8E{PtW4$ff0vj(fe8|i{KLV|+z1MC&+MmxK0`wcxBIX{LFxXmJugAE) zze}{)*)+TKdL>!^9;=%m_4;j2m_1t+VJM!Ca7=TwD2$eHK{Om6ic>=PBF54F1=dNKSgd zcVMISOC|MWu0gWvQP<4-pZv6hujc=Tf?h=wLt5ZO2>i;yQq!$rS_;UxvtH<9hS!7G zQ_13G34bgPt6@Vh{;lVzMGIuLp^3;6-wiz82R=!KQ&ez*%N@eC3kVhBTJ4S>09ytA z9P+gO$l5n{gp1)r#nr{ae%)R=7-l&Iz7#%gh6j3YGDv6*iy38_lI$F3sXq;XnR0=K z2#HC!<1}RZ@|Wmo@;ssnA$8$7`R$-U6d``|6sZN<=|jWO#6rT(<>htHTrP|EXE7xb zjc*j`*g7{@v)}V&4}nGDsid>eObNuCa-7@J7seoU?lB4fSRKZ71Iauoy-<3O-`n)I zbgt$mCwWCubj|Lq?cv)@5!5bRJ&k6lubBmTO}30sWA&7rv`W##w2B}3ddh1|@$-iJ z`}{5&4Fn0|N+jC(Yw$7H?mq0t40TdFR0VARE&r8G3#4^| zBOfeqf5BSKN0U}PYTB_#G+!6|`mC=rLsj04vP{V5?~31?T3b=8qJ93o3q|js8}>0X zMx}Y+*8$Citp>*1!ReLhg+=D2F~^@YhjX60%CaQ!H^ab*NSEa1bUWfpj2_A!bmv(u zr=vy4jJnaf>-GGlYDcV5E$E#`H~TTRT*x3{Y3>TGS0egLKj0u$r)XRnoGNq>(7pCJZr$B3&V9=59tI z@UL(dhlncm2JOLfflXrT46MC3VT_5X+&U^>|qSjaOPLflk_0)~*k3;qy%)lEpnE+|ac9TAz znu<~V=6(BR;g|5DIcoY7hix;W9Ztp>5RXKA*daxU5||Ox5bjgP*G>MfeP4A+xfqLH z@>1YFZfx$$64NisYUzmJ-@JHWch(WEVNQOI5!pwYA^1|hd1zPm{*alNFu8Hd_#xFB zJnc&763>z9Vfomx85X6z`P8J$@hKTCkv7?5(Oo^vwbH}g@pn~tiYOa~eX;u@h^qF% z(!&kxF=9YsDi6QBo-VQ;H}q>t$>_1&@za9pow*OemT9&reKZFj9o#dZu6@4lVa`;5Keuo~i4UbF(G* z>-fFY+{n_6=~7BDkh)bA>Y<~||M0R(>_1eUVgxSZB~x=c-!J2r z`!C&tseJs-kM#(*GuG&D4;nV`^hrP!BG=)+81&_TF=)9Yob9R25>@TKybHdC-TaN! zDJ_@)gzQOgy5(Pq36G;Pzd36~7uC!YG7den3zEwGI|t_h0iB`dUxxq`X#;d__LTOt zb_|dtCc#gag~q#tv#p{XEQ$45xFAwUe&18t5iuz@=oL2-xM-^oNgHPIL0mMjxE zY|b$@pQD?;J-TgrAEG!aQnAm1T5aZH6FIbVSm$glT6a%swod-MyKx2lJiyAF{^106 zI0H6EJlL+D`k#~p?|7n4E#_X@g+n*h(94S7S8u#BjCh`x`DCKfy76Ab8Mls>HLv}< zyj@Ii%u5L>E3f?E=SlVDpy~C!sP0%T=Ni&wPnhn~mo`xlUg6i1Ky+;4bUbS7{OATa zU(*W#ZvJmNG9H|+4J{dWET7P3IuC@M$H*T^7Fq>5Ac5^Xdta1@ks;Nx(pVGfa^qZJ zHBIjm5o`x2?ECcTxOr;&!+Dac3*+0SLS~TJOY%zPh!ppgmlX zM+ZwfHx_HMPkuN;#EeU9CZXx_l;QYW96B=C*GzgPka8va+DsaFywCL9<O`S{57)PX3cSZxetocbNi-}{hvRrgD(as z%Sh7l94$7X0TMd?SIYrR0*2(_#vEIsh)$tsE!IRH`uCUB7gw>H5n6af1`yojjV@3K z>Y~S%M%r*oFfm2i))C=HoFAmOmZ7wW*@re(5YU7vLI?I9g@7!N{%_CQgeO4;CLJ_l zBrcp=_~+1mSg|7n-dwB)n_#yR`f+ENNIfX3v;$PiBn8_knKxq*&Emad_^ zQHKm>_V10Ix@Yzl5T|RTavXcbC=z5bNPkKJN&Mr*zG%xlYg=?TL;)pISv}R{`EHag zMYTl~S#04)@x zked3f`|gcI68ALazhs`M{F*Uxe;35`_0*$-f$mN`P&P{JkWYe6iTZL6-A{eB)va0R z*I7_yBK1`rz0dr%_i+FiJ<&b44CVacG8HjacQ4CXK*)%8p61N`z|F;#ajU;^QTT`? zwFha6wFrCGMz3c_&e11t>tJFpg%QIbK=1QOzfmB7;nkZN%9ysOa`o81(&~qxP!{Ze zzuM>{x<)C_nDOQuCUW?U;oC-!!$F9Q3+871I(q6cvk=hV(p8VpmsGa^9V3-Myw6#S z%~c|MQ6f|6#Mr^CzdL=?lNe;iegXp(%1apdn1jvmBff7Ic>PFZy#6f<84zi|=vHU^ zg%DqZ|E$_602Sp^xtN>GOb&k_UC6@bX^k(J9;2{uM z+ubGkL;W8fNxnRAIenG9+gnYNCVRhgI*W)%D0RGdpW*CawEJ#ZGSY+vEYw_-6#ZuV z_`lp@5pnx}j{~?01dM|>{qyXXHfC&vZ;w?U*8%)_f3>%h(6U8FsK!m1n-ITKLhn+` zWcVPl7AP{k%y|`5MNtNGhU)Bk6x#F8-29nDd7vTQ$C0;)yo$tNA+#0Ez51AQsaxJsb5+)Zn()!6 zY-9WEl3Cou81nMI{OH(p{5lJ*jzqmy zOUqp2I^y6*Z{X=Dy5Ab}x#uq;uTIPRb^H0OChwaf>~TV4YYbG3xYG# zW$&XVj&TljY|UJ8Hx?2zOT0oAA)fH^Hra^2J<*9JDk(`v#GO&KzXXP?w@E&g5$ zxSI>fWJ*^uFR=`eYL;?Pv~#5G7M)JzZ%>6^1$kgw#W%yq45nWu%jk5|%WIpZr;R1j z{4DKq%dVX*&)3cqrO9F6-K!qw&!}HTXx;&KOw=E0_8KFJQt0YCaU$4OUZLu}9#M14 z>E?hDtuSBj?U5y=CrnA{RPBPmV4jLwuzodn&5U*QkCkeMG-VD2pLm$TLbbMg_nq*v zYnzozZKmjDG9t)YH>^;Za3+GvF^s)3TWS8%|1H0v%p>r z!CSDdjA}VJpDj$4>hEkIq*!|ZS^FWd`!_elAkx+92dS}hHRy@J(cl}b$5;HSOPirC~UVtOrWd2IQW8h=bPtV4@W(Jr4a*|UG@$taACn~fSvVnL?w zH!>wQA2ZK3YI5_ugRRqChP2g^XkQNJdAAbK!PDuzF2j-VX2l8YRMbhG1vQ$%191(2 zEX@JuL4KJGt;0g;t$`m8uPP9{$#Tf7&SD}h#Ovc>eQtgDRxxcvoWqRMqhXTVEpN7{ z^S8XU3rVS_Y1_cJ74(W^Y?{r$VavO(2JJD|mHk9t0T!XCNy@#FT%vYJLZ)}XW6R>Z zrRy4Nx5x+d5BV(%X^7yPxHaF+{=obzmBun}NRyf~gND*ZPtA__6H5+{K?Sl#q%ZjC z-=MJ~&L2ghu!Rw+iUJJ2L{j=x8omqYDPr&9qMIY={EybtZP8}}PdwCqYpb9)Q?mE( z>BU^w>D=3k^1L8dKlTMA(rv9lDKZy8Wh@1r$YHa*8yuy7cP0YlaI_tMxmoxB&H|v%7xwLRN#NT4 z_L&UU6zRt62-vljMX*0{c;^toQuBOYzC+GK`mdmY1aTM<*ZpMoGEAYu>Q)gT(!V6) z-v)+J{E(Cq%|foKVQ*4I#Z_|TXIexro4e1BIR8`MPK-yAGm91MDtW3{oq&3TIapjs znNK2l0nT15v-yWV;ehDfJE{nxG>3N|q$gFjl~iQAC@^q(Vy!2M za^Ll4!ZMajazc{Mc*J}@C9^@g*X~K1Hyu;#U0GRNqmw}!G&bZ%c>n`>kcff4_qZG} z4mgU`+N1kMZHdNodOPBA>*U>1vy-J!`IQS58r{*@Silb%QPmiNM+@B(zgmj`TOeOr z4xw#-BD9)Ck0)$FGZeuzd;conumF*Pr2fY5zHF<+QervtODqsH_zbvIf3#hbq^~&^ z;9$Zo!tzm--w>Z5lDi)o+LMub`VzTMoM~AYCFTxQKJGpYudZs^2s|CBnU;&`=J8Ty zg|H;0Xh)x?uLqvjo>%@9Xu-TwYBo~3E-tjk@3`(A-p@I8|0=dlFIBQ5h=4aPFJu3#Wh_QdwstHTFZGcElrr`Gxj&rsD&nsrF!{q;%wv<1qn= z^|w~fb9S^!zj)OO9%N{QZ}6RhYPt2~{Hc-{+`?pixvB-%vaUYoJTnF0sg=2BEHrL< z{~bKsQU7CA4F^!P+7QVdkkcfF9{<)N?qITR(JFMlN(KU%D`0RmxeXspu-%>_{c~Ia zPd%7IvF?z$w)*Svzh6q7=gpA|_rDR(`bjds0ZAlp$M06%c!0rs|0J_p3k(`EwYL@l#OUY z={5=ZpZ^{(>yovg*EE@wXz zI}t<`m$Zvx172{kJ-tkD$aDPZxdRwfrv*ydR9qRyCOXZiUnjq3o&;|tnYr?sNAe%T zV}~meu2{}ts0E`n4T#8bZe5#rk5G&=WDV_?qJH0LxQ>1?Ssu|+YqMD=WbR@5D_|1H zGX^zAvWw5Hz3cffT;1GwtzK-scKuD_Ow0DgD}f`QQSm!J=9Vavz0Loj>n(%Y`l5$j zplET|5-9EzcP;J|cZ$>EuEE{iz0l%CiWGNuDeex%B@mLE{@(k)ekb2D6PYvn?6dY- z>vD-RCiS1d8D7zV^>$E!2GG;W)5j zd;ksK^0L!2)2}F98vJ_D%uh?J4>}@wf6m&VXEgR196uvzs=OXirf;HU}gBrz3)53-cuny=oh5jFTWh>pQ z{M81>?!KqAW=$66AC#xv|Nh`^Q;OHc46<|gyt>l3d4ou2MB;7&bU?jRl;Eh&Tv0UV zi}_8r4P4TiM=9|1kOmpxrvaEJ;X*wnq8huk#C|r1?3y}B%?BbUmSoeSQGUxsp=RMd zYt7Qa6khyg&;eqYHQg0YN=;}KYHCfES&1%HAp=Ie>sKTLD&Ufet?ye-^4r0v{!A z2IV+Ze@>K5Om4$={s+K{33Zq|nH?G#lQ_G{^KwoSIjW?jVo`F`sM`=JP=Du7Il|8g zTTF45sbW3Onk9Bx(_r`Zk>RXs$p<8C9+%Ikh#?oI+rVId`@#1;z0b4|Si5Hr=>N&^ z{=G=#i0ilZD-3<9bPjoMNk(k!k^V4OE9&?X9K1Bix1yII{*0qf$S2QuyuJLq;$|~$ zf^wfeEe@4>ez%5VfQgglTp=-mP&dHIt5XmFu_8fy$B0j_9PG0AN7rwRaAZkxl-sv0_yT%wPiSh8L&AkQwt3g8rqE9L~+kHS^n9H?CPOEAn_C6#k%I!=0 z--#YjfS&f?8?X~-MNwbanxyCODqmOSXMS&9gi5u7gh18k2aDaE)Z`HG(an`8|ER3^ zmc&%s6PXYA#Ac!AJar}zw6gkcQMQkle}fHM1T1j+srsHZG$~Tnq3sX7qg{oyBGQX( zgi+j{LItH4&!2HLMYGbf2G`nkxAFJ}zstWd_;sEN)jv&}hNEpCaq^r0?=bZ}9(Joc zVMxj>m*XyhVyl+=9CD860A?g|O|W%Hpw+vbC1-)S3|A%R8dUJsAJ{Rv+hrIkGYCaM zJ-lOO^InTe{VbwR;`2^KE!()HdjOXFRM>OM0dh)}gS^^|fo9{SWOsn;W6T7`J`8{Li5YrdU+fi{1?bz&~uv+wKPa$!}w%d|^-f?PWpGB1I?P@$t( zPH1`E4e<%0&Q20}{vWMocgHj*LfB*(er8joM1V!D+JFzZ5wd=30#^r|6+=;M&=~>+Z-#dlBnPN3+_$h7>|<<++=Ka!+gTpH~`Qo zm$<1O9>9tb_BTKtK+5lkaP9(DRTaM`**{6V<*Zp|1Y!LWwJqKEzvyl;lRMaNoL;`U z{CzKETX}xY126&>*L)QW=V>zi;JctFcp3A>l9fl#pmHe5gAS8pD(l0i_C4QeaU)-L zL;;c0IfUhtK;}&8DocU{@C^M}o?!ZuJw|{3NLnfyuUtXk6Z>{Wv;B|r=QWxI)LO?3 zJuY~CKe?a|sW#-6@g}D&$pRqQq?6g_eO%>(=z%f_k&KR+F&N&ElBReASBwB$|I;1? z$$&{sHsg}HWkXtwY!X%+o8e9B<=WA73 zrFu+i9-46!;6Wd~6Tx?A;>7juN`@^-DE270DKn%trB}rFP4TBjBmv(>Njcjz33UWX0whgXVR;~2 zbR;iespsFSwc9BqqiZ^lHjl0V61E0IY~Aw^M7|`yGEWEf5b?pf+4x16xbUIh{`E|f zu=U_uqQ4V)3T>X?GJyVr2`PcCZo9L&*i?+Lffz>Eam{tJ0C5W8_>}yMCGtSCd4Vp` zfL#?bgSl7TlJ4|eh)xrvzo<pKA9%p z3>Z1Ama9oUlyaAS`lIR`WjHHOaVPX`NRy|ur zA$P<0z5c|kNX>ZCoY|Hp1THz}2(lU6_BPjv@b= zB-e^BW@`$c1ZC9JFekBt!yZ2h>!o2R+#@@nZe`92$#as84=FlU`tvCpPEzfgQBbKA z1Ere)4|n-kVB|Y&aD13(>g4|F_*7`UEv2)0A5x&x;ZV%-RfgoAWuuC)v{D6W!ltsu zn>Ijz2+pvbuk?&dqy5250>)a;VdGZE&cY~XSNkRPaL@O1SPdl6*y?h_Z~TW%ew6`D z%ihz3BdkZ<$E}LG$exLR6JD4HTMV&3CEWD{>DB61Md-|h>_M1;R}1x`)Yl60yaa6I zbV-f18Tw%vUsV{T-p$4Gh-f7|1h3~NOYQF-(}MG_>XWU9qQA$JA1OMo$>?0()YizyVX^ z7qwV7TdjH?DDNPsj+C$9*kPLDcr;wZ0~R+zr|{-w(Xvx=dERn3ScQ*>tAy-SP->k4dA-HH?N$@00%oPdNQ@=CD8AIx=eMPvuMenuoQuKAfhXz&Z zZ<>d&=SQ@zPW1gW2=S=fbazlG#qY6C*swOeCOGf1BVpUL^EdDqbx#MlNrCl1snSZe zVuwB!`xVGL-wPTG*?1V<13O@oUDG+nth`9P)wbUsk;d-RC~~_aTSq!(txHbk{ZNHs zP>3X~yBYyUm^=QO3?NIt*Gk7edZSRex0?oa%7J48Ne2=p;% z)@%rI9e>N*a;oOP1AH}yW{c+r!ZIUk+qWga1AJ!7IeQ@W-k*RC) zlAD&^lA}849)OK(fMu7oW$#=r{6t-zvyNob&zWvhDMMP~T>P~HvZfZU>GBMc&hclT;xZlh% zSEpVhM8G6<;!qzr8ysP$dpqnS55{QfhP!;}{xAes;!jV6d?~vY-w}sJwf{Ze?egxu z$bjTt-~+ji_tnJW6asp@ir?AngmEao4G(KjcAltM)Vi@;Unf-su zIaiY+#ppx56t#P`dQzt#HNblf6_Oxk8GDKqMa;FZOw=uU9VzY=f7qMnrXlp*79}JX zJfJIN*tF#7!;{}AqZgrL= z;0CWc`4(6n2b+pIM#e!+ZuS30~iAjq}sUjoP*DKFn=Fu_E%! z5Q#wdV%ze(hwac!zqg6MsaOb0aKtlB0#er>XJ{Hg9Cj0TSSPJRr^Uz}?^~u7Y(__B zOUwM0^Y^nO%}Nnh3E6Upx8H7!&XEu63`wfHo+n3+YiEKtze$^xh*=h0*i8^F<39!$ zeRN2JD=GP$bKbsH%th+Sr(?IK`nP*ecEx&pl9%m+>u^BgNY>?0AJs{=uSjV`sL+(P zb6wq=4y?QOE!oBuG{(eSm+jA-S8l>{0FPt%xqJTZsS_wv^372XjatyqBn7G?R?0Ys z0J&Z6=pt;3%cV+`{9Ue&C1?w5+XXjLw-(jl&Y~TS9+@PyIM-b9W)?8yFX_wizot5i zUVUIFqytC_G-KEP4LDB7g(cjiO2Cn;Yy?;sJq}_Wl2M+z@)vH06B(ag@~Msr17ha5 z*82GTd3VWc7!nCh{UiW%gUwFTjVcFxl1+9 z8B08huvX#h3hSZOhLHb3y&0alyOh8--tv3cHhfqE6z|%w>h&Xw%^|^6VoR|26;`mJ zs!Vr=92rdf4~!k?e6uc_8!!6X5xeH|@B0XEMexyKB}{vAO*~P=O*o7>rM+G%AQ)YN zmp>v9Jy>laMttZcO)ay5_4^*#j)!z*rd)bsW;fEwG2_Skq3Buhj86Ku9CdU_H$0(}nEI8Hs9 z2YMa&P1yX-=Zu>b-WHoTjt54LI@`K7u{+iLV)6Xr_~is`jq*a*j-h4aC28IHOcV!; zK>m^4ct0q>D^rt#8)Uxw(Fa*7goE0wsCaH`n<(4HY%TN%fKFvrTr&4d__9_P^eb32 zZNu*|?c&!4ndW(|CaUC{HE6E!<6 zp)JwRfbLaK7NhYs1MB{wvH70qdUMm#&D(>W-%b(QSNS}z5+t#r=UMX-a_OxHUI7C>gTtj!a(FH35Ed!rRrzJ1X>-|FveVv@n{mc^6s~NA~ftzqJ}x(a@l+ zbo)5HJ7=2aPOUngoP64+W-Hz&&!9y{Dzqz1D(YFXx!ySX{N;pqa#3K((xBKfYj6)( zg@GQbG#nQ34wjo6kmv*cuLIc5*TNywn9#2f(}y$Dgf2U$Acv0&=8dCzoxGESbeyq3A zw@aAUTkMb+Fr*~ZE$!k`>|R_HrZE!cF&e}LpHh6mv9bGs2CvN;?q9#O(tHXumno^hFbW^12E zlBQKy5Pa0lI6dr<5J?X-BWUpUhSUg51 zb`2&K_xm-(g~{5w7ex*$t`#PnL(lnl;X|Wrz*em~qQly@lFtzoUZ*QuSM$^E_$tNl ziAhaiNs$Q2_sfR5j@{SHD|otl2=Fr_w#7Lgn%nWesau39@tEBE?Y(;(L+9wK^pySn zQxZ%-NF>Q!rucT2y&zxsIOugh0w#LR#Ye=5m)dsTDpixzv*driGBD{&bv2OqR7!W= z`nCOp-WBOQ9PGQsmR8P~CU~Be>Y{xKw*su|EQ)p`e-5(BqL1O*d{s{hJw3V3g-rv{ zV()%HZlz9JwPWx(hZxQR!+jCMpUTvOQ9)Q_yu+nxT|xPTyT0rR^H->9+;^rL<{uI+ z({?;dY3>xwH)G~YirTgmgKzRDxFw%nugk!_aUK6fA0Rh{I;RpZH&4HIw&Kq~%y3<>b*!D4%#K`+9WCx>mK3GVB#qCvyM>N`SF5v=|-oe2Ny@QG|*-HW)iX7{YKN)mjgM6-a7uqS7)mzd#=acQ8n~AN&Nt z+i#rNE_YTr2$0noIZ?=n_&4s0z_*=H2^{rr2#F=T2rFitssf&Fzp8&v8= z{Sr6TWeXuT!#U^5MQy*t`#y1C=Pjpgs*&rY3K!aNhN{pZZvVEeD4Dd;oHcVealNc2ji5afjRKOiJ^W;< zjsgOreXORtn`k1QJ;U6=#tQ{2g@XcLzQ8(bDW&-kjdz* zBYK9I11HXdVYQL#CGzUFIdDZOU8H^P?TN}Q&;a`JtQ+Q|(8m^P|4H1!)lM6to0+7j zerTjuiS${{>8oCYKP#IX_u6}QHdF6~3MxiS%6D`5rbtvoQOs00*U!7}Wjcwh(Xyky zal%qstVt;S5J^J5pLFUnXAK@UENb)C!&puJ@uN`vh6fa*1DoM7mK*RIoNItfi4032 zI-i(Pn&8*nq3mcb%wZ{=P~>4|hYKOI*iY(GRnn<_3V_5Qb@g7BAdoY@nju}DrBCWu zRC4Jz(|Lhv85#N;0 zKg|2w@r&5F?D{O3!pn!ImrOpn4B|zRZ$?bZ? zBi%a9Ff>_q4cUK1d0B~Ty{X(b>ZoUhNrpH&`~m$pE6|7*##n)gpbaO)Tzu8U#9OEN z=i;7()EDG&TzL#8^4ho6^-`YpqxDbC4f=C~92Fi2%36)<^FVk|4z zRMe%O^uC+<*zy@*?d8*aJP>xJzBx_BM@w8n=MWHVvsS}0)B0-Sok7)#T1;w>Iwm*t zRUETYlY|hRyfyjP!+Y6r8=30x>eRQ0SBDxD{oLF4xtZy`cauM7ue6Wi1x320#hWFv zD#(VL6{+(=IhyIR)G|0ag67{=J-qGBMt4Yc-jNY;NsO!MGI3LK!0OI6XKuC#!5}F= zct7!Ip$zPLasM!!g9BYCkOYa;mfg!_vqocf!i3(qY) z!T{!9(Z|a;L-|FE;T7eXRU^ilICym?IL=VZi^0Z)gCcLsr`g$C?6Yv6iY^H*VyEO}z$*x!>2?yw zUkZ||n9Qs2)+D2v($g?y5@^)q$|O_}-)^SJki&M40LFiSH64%xpk*Ep;=FPYP$BjK zus<>s+u2XYJovoXP))yT&%nM>ZkSE2wG1GJ3xk?JStF?8L9I$v54l2$#s5Q@Qjh;g zewM$*ZhJr3%@@ijYL-+_N-xlNSF$HBIw^kp6T0*B0$YsqAJAl>GRO;9PBOCDS%R5z z&R6>Zoz#=7jOAxMLi6CrKs0%3sXGqxd+tng_IN04J0Rv!bZtu)>%JH@lEs;=`%M3T zs!lGKisU26<(_#Ty_ZSWGvt-DrNYrMRJ#__%8Wh8onC4RN-aKK11SHB!P_4$6XNo>>LtlhWJ~Xum}!n0#`(C*mM2Y!!fqmGb#~K z*mCd%0dq|D4_$;?ginjw$tQi$-5WHeJ9VmF_E}dA{f&$xqMB?gLH*55wcZH3h?z~4 zT}F!8b{Ym85?&+!eD3N-dd6G_I#nhY2?Khk-VKpcOK%I`syL2mMSi^@J#X+C_0yCJ zVdf>QMxl=_MhRUQFq`f2zlR%o37B#j6=St*Cpn1ZY382&4mP`N@dAGlB%S&YY~flc z8;>`W@e9cGWNOz>IHCKW%mT-fX0I^EuCf`KQWSItxmnuUO7@&-Ex{}@bOML2p8WWp z#f~H}UZY$MK0Cn&rygVHy-`IpA@YK#+-*JGZT*3DW}&Ac1A6){cRbXLhe6wkjLX|j9GUJ`FR6F=7!Nwmyu zl;CiFy00~r2#H%q(4!F~uM{^V;WTnco9P{-hupcGaJ~P?$i_cKvR>u9=&d(pyD^^C zdtXwq_Gy_ASD-tyGrZ-@8tS}{CCXghcE$H?zsY_lu~%L6{tYHv&yI>WEYg_nCZprF zo=o+0Fc?iR{2Hd^tNG4nkjZ!ja&Wf)TJ6m^nhrpbaaLU~b!eu=EfS&u%4Z&ZN5VrX zyCEF@pos9y1ygjfB52v{W1rkmG)wFTg+ZVDx)tq$eQ!SVHg%Ox7F5X%*%oEV9SLBu zqCr1_qNJm43cZpxaF=jkiD)3G7wS)e&&p;;M;AV6wfvfpRM$nu{%6!RU@dPgJAzwX(C3TVGX(m&gl9a!xaKm@VJ#Gc<3m@zUf3 z+})7G$7e-xEKo1Y=ndFuE1Tx);Rv{`6p^%^kH%K3ex+)!wJd&ACSJ>xC|a5b+!?@M z?=|1ih;y^HL+gb(ROs&5p$|AK!sGoF*!8_T#iei_IfdQ+{?9OsA{<5q;#6wQ;i*kPBee` z*TBEG2Z)IL3h7<%Gi)(vwx@&HWz%4UMvLpC1OI4kXS@8M7Jk~o<3Gsv`bGR2xL5NZ z`ltq8_683(e?sFq-U0-j1$V~dRb#fvma(-`E-;?J;>O#`dm3$+;n52U1K|k2fIpGb zSyFBoA)z2V9kUPpGNxu0fT{To!*wUJ@Jsq4vUio)(408tWc13JQe+Ittb=a|kvY=u zJ}}8JdP{lg$c>W)uWQX_v4qh+4EhE)4H32>A@Qt6J0sc1ab2?JCIM~_8woDz@`-O^ z*8qg)xf-xZ!!lKU`+``$B@C9keW!OX5|2{>GErAPa6L)s+LJc8gf&y(E1_4hnNBy; zAO4AyiEHU$5L?D9DW#ip9wQr-zva#YJY6HDFmX!EP&KSxqF7IvSm0{`G5n7^rvhNf z{fQ3)&f!ge&1_*$7kv;nl8qyweq0Z1JfG`~RnGtL#(vcEZb*0jIl<#F^_b(@lnglc zPiY2Y2=njdtYJylH?WC7eu@fjUz3XXt;|^ITr2y|6xi|{m+z={J%^oi2s>V{BmJ)xB(!SNrxBu!)k@A zGs^uMJk133sBgH_nk24l!OS_eejZx&tx1~WN4Ci?PaUtq{7e4&Mu%gO^Z%()n74#@ zuw5vnA^F5Lo2=|^2mN-+HOGupJKY*rbp~eJ8k?%TcB>daDh;S=6k~CuKF~kQ#UtC0 zrw{G`AJ`kWCH`xQ2ZDOUUxICZ8D^%B{_Pf3?z)EEgu)7gP5PcSEjQhD9HZ7J|Mo=6 z!v6|TS7)#iKr~cm2~-F`iFvzG!|&7;N{8cd`=M?}&Z$#_oQ~)4ZRimsF9HD$^^ynr z4Ft4<`90>I(2bnKvmQCal=lk#yRlm|7@v->iXgsb=+4Fec~VTnFZAML(M`#-(fZPS zj1#Ujb86lv%lDMTVp}=xIXl)UTaU3`$&lr;W=yN-Rn19}xbaG&1ru#gadk?i%j}ES zDpCt7io1l404x!to5foe0gJg5w%JU zFW?5pxU+RNovYv-f3Is1n1ow8te_#OwDSFv91|N$1hq9q;rv++Q6V&aR5>MPpjP+p z+lc8QY)1rmo2XvZkv?(s36}Y9-C0wlz^6!Y<#q z8k7OD9IU3G=`srko6ro6hGb@51FiOhb==2KHv`LQ}p*m=Elj#qzyB(%OxtKPc#9DzY(%@nw7X1FLE57}RBT=$}@(_Ko17d6B2_ zAG1C2MEBY6UOs}bUH|To998je=N|pZ3cj3E`INdvZ8f37SN#bXhezH5#fRWSJP_<) z8lzd$lJ1Cd@f?yOB|= z=l!3KglD83xF{Qa_`?@~fP|IHk(U-^N$TXT%-q+J6osLZ32%07Z(7L9kRC;%r=wIw z5L-GP$dyILnl`Q!Qxu~U#HErY#mPeJ`=#hrFF}mzMn)Czd%&qM6DUV|fGpEBLK@7G zJ%4ntrNQ1NnatrTyRCG1=L7OeQ*u<_kIlnnSw*CK~ zkA5`i0LJ84g?3ZYi7QtJz1Md;=?IkjYOfTIpjsKn&IWb6vz_>-G-!Pq{y{!Lcl!-P z&gmgYc66DhEBS>AR(Xfze^e`q2JF(3er5LldrG*xz$$n<;_*;N{dLr{%K;$fH1PkW zH(7`2P3mh;Wh52&VLnWiPue z5UmeomGP$qCr-4$&PO<;JA^9-?l;aiZwGeO=yDx7m*GAUj^6MK#vJ~Txk0DfUR;7l zVkM}RaF4mrW-J_fKb}{F2q(m8Yup6y@w$L(D3?^eaK*?a5Zd3YX1Mp0pzoAflz2|k za^--2_P_UO($d2h^rh0AFtW#VF&2VTSpRdF$UmW!bl}X#EI2)loCEs0ST z%Xg1eZV^b_=wx=C{;$iiv-~GJ8#xSHzso$XD?Ul&@A4t1KeYH}w6P3;SvT{Ye#RaeN~5uWJ^K55f^q8FJtpS559zJyHm$?TTom$BxSeRWGd z(4tUWkH8xgP_XFvfxhNob|-Gb5`*Zh$>T6bB}v4Cjz@)jUV^-f;F$l5qK#LQG0?=&=`-H|MKqx3VbeWy>bOt6#L%e>eE5V?v&3YXPX=bZop#HI zOtn8$NlsASFDR6WFLu)V!yUE8o4z!c49u@)5Spe_uesWw_M(LTA|V&5oGujd`5W3d zutRY=@I(8z(isoC^5{pf5hHt1E2Xz4%I_iVvNCcqD^;%-<0B7~g>vl=<8`f^1Ag zlJBNa-&37j*2HxW85T{plFLDLe_N(b-Zi1=JO+_FBQ@|K+}0p!6ea(=&V%otx3OQlbbSt2^B%$+v=! z073d436f_UaRKe<;ts0;GXl~nok;zWjdZ%V_#Q*qFniU}e#D_{Bct`!@{m74M%Xl6_P)~Xn`7RZ`w&2RV)<9#ITXEBL zYOPtiN>1AiSMZ=q8`5$`QVJfukr7-&gwABK(rql|I*Gn0gW`d=-Dzr@<=l!UnZ=f5 zba*7MPF%)+PnDmQAWEQHqWiiu;0QiqjF`m?>_?CIAc8LrCj6+Ib=-=pF?jf;Cb6r& zx1+g_wr6%hPIbU@n!!)6k64rD ziJ2}*P9 z{8=3Kx<>S-!U=45kkI(LHw4TOH-Rbj7$Bb@0|NU1*;!W+weS5Q>Cq(RQ+%N5#5cF? z9lMao!l<+0fTdj8keS~5o|dhW8h)XQFxa(C)QHos_yS4a+j_b$OoLWvK-@km1cn)Exv%zN^DDue(kU<|p^kUA+y`AK0FIZxJ8^iVk|g_O zY9AQdJ8K#B1*oN?4;2I)j3fxCI{-A%|bxLaOR#h|N9A zUm06vwkms9~DnFe%XrZC3w@gkI}DM&8V@7 zN1_H^@$#ozN@!bgou4+U(f;%N4Yzmaj=i;R6D4 zuz+snhdb;I-W4H9w#2A8;c;v(($7NI9C=^ypYb_rM>JiJM`mx7yLeI-`ZGfR03BsFBztK)n3guV*S>c-#~c4y^=d|Jnrm z=<3vy!*}_Jb8g$b94PS0D+>JAI(!l4DUphyI(d2aE zKL#fkS{Q;VmD1*u6JBu*xGSlX&$chZe47zaGz9qHKxt$wL1u#2-IyoXfA1^t7B9-Y z?`AYMG6r=}G5r5>1+Z*&6G9@D@+acVkWn+>P`u4oRMN!tAy3yFXHma>6+_3NSK`q5 zo0Vzk&!OP(hO%$N&}UP~{EeH}x{iskv${>1a^b@@!wV4nS_lS|)ONDLeS&-gPhVeF z1E3%_@5n>pN6~?$+-YAK1w*FMrQA-wG~2Vu5Xa7BF8B5$Q$fhVlNVonH(%Po0Cic# zR)=?%a>u9*ZM4X`@7^HBRK1mDVys2V+^;+LgEf+p&$?E~p(r#}TxEOxYogR4X{qV7 z)~a3mp=C2ET&muXO2-Ag83%shCY#H{M2IEpiEarK-i?4=*=1hXvTdHzT~flZf~*Lx zn15aL-?uf^nY?iQ3unLOQsTUq`s$+Td*r8YgtdZxokN=k8YM?iRpPyu==I)pvTp9H zKYSb1G+FK)KWBKzSRGS#iiDbMwFDb0CSrQn>0Y3O1%%j3qQM0Q~z2e&2_ z^zRnLX?z+o(R`4E@jTBp(fqTllyrpeL`}Zu&)w4SRvGkq9w;}iGL0rPd^x$)-2iQb zrb2QIHV72HWE%DX^G12a#j6PE61`Xyzaw>HvMJ84`pp-(RycMFYD_bZUaLjn_ImQ< z?g3B5-@H0WDWw$0_u@}bz_pIVBGzh9}LeSm@3sC9=05p(HQe!PKWRS&hVmkh?n!JV) z)2H?W>lQ4}{FL+MDdnwp0>sGg8bN0m#TbQU;`Q+Mn-fpDcjAH1fC@l!uh61f-HD4T|eQ7Ede>FAG$H1#B`>a_$FD0Sr(~n0$ zyKJkxGow>_T_#S$?9C10ur0;Y9xF`i7KFsfK=}a${orALYZDd*fcNpWYvvT}+LF?J zC}%rHVcPQTkHI@Jq`1r31N8eQz&P2kUks{tF{a1BoCTbDq}9%6EB3eXKXXH?BlHh! z&M*^W>5`3-`PM)5b4MKs)gmia;2~1rk8x6dBvd#uI$f8CNa_@CwewUM>KowYWnd_w zc09cg3Q22X7iaK6>LF&z+tTTsR%Ov4226KRsBe>@PZ8 z6rJcum;7x9w$01c8*kyN4AQyO7|}4T(lrYsSk3!rY!CUe#+Zc3^}tu2+*QmPMAOLY zgKJ#K2vFn2(Q^HA?Vpby)CcTe;uI@uEkfij>s}fP()RB+ts%QNFc#A|6a7WJ1OW^Q zvn;Men%6J`QTg3-U77UM($v@qp=T}i@Ioq&m94JA%e~@~PIpZY7C592P)=jZ(nnd4 z_VbsgOY1C0ynVSWyWSpF7H=?=)8>0E6pz48 zN82;XqA0ObBtdVdKv}^qC)X9s+*nD%5Oog%4no~PmQJ?KodO<1-9K#Q{-1b}uKJyobqUSGxmpuD4@0TZRv&HXEzMEjHTa0X^Cp4XE<-%F3W zuplc(b<~72w&h#W=8HJ|`f39hbw~U9!&&`*Tmaz<_qTH0kRP?2A_~q~h%DU91K8hi z8q3hgUuH)4{`}lK2-0aYa)%xA`izAGylLHBFwpkfzK+MNbG`MypYAd=S+Wf9jHnR| z2*9yUBW7V$YA~WavWG6>U#D4) z&x)gN=IaGO^jAo%KE`f~DC0;+7ip4m4mSk~L00m(Xd&GPg9kSU424W` zi2&Ws7dK|I&7g17)=j(HIaM4SM;EKbu4%eenXdgkq2d0?9bM1Lf+ToyfWFrE_o)fmRF zEf~f^P5t1upl>!+aZ4sJ*F}ca9Yp_$k=3be zKYN!HvPf-4e7@K#+kJj~1$60{c>8qYacd+HBJMAaF^J;Bn7!};SRO9xj!UEF zKo~fQT)|@wJVALGAAeRe*w+#mVG}`Zy7ht?!aL7OxLPdO>3fyMFn{7jVvebpOhdO9 zZ*Eu&sEd`=;qnF>=yxYX4kS4XTl{Q5K|1dzzGzZh(AE(F1bUOMI&^|NZdT7?lC!i- z%I-lFL5yGMkBIZsOo%hvNL9cNJhnc`bgkDva}R!VbZVNc)Dhw-$^`}*F$8}%<+L_( zhB($Bw8`5b+3s>c5ts8&SFfJ5W#R~?MWLY$(EI+kUJnU%fMA->xFtT1{*2-)qp>bvd03)ALydD6R(8%%Dyra5XceF|Iw zsCdR@y{Nky=JjMHUS7nCIF-qwGoYJy@VPg_T*KXE)3 z0n|alb^fWd?M)bU^${&hV=F`Vj+EtF8ZkH2Pasp@Se+U^j(eiSG}ZP^(hldI;)FOf|YCtmZz*v{42@%0TUX;?^g!)bo7+ z5s%}(D~K)>YPoTieMdk3o1;y_!>{6$-G45bD?x*ryb4hc@HSRRMANP(+11YJbf6Iz zI!r1bp^j%mk4>8VJ>nC-6@`y>7Lk)vM&XkhEO(arM;?@VlWAW1nR*yNLo;lOhFsVi z6%@o+#;+GHQG6e|UC;F)iuQdFzY0s&QxQ>;P-F=hSk_~C|55u?9(@W>oK zZyuq9F5MAv)=;^AcPx1w0RR_q)I}8mHk>>XbICRICU|iE7rHgSwTXgS^#tWP_#dzW ze;y%Yja?J0Id{Mv{XRGC*8j0>di3SE-v8*EtA2FvDKdkbnCt0y$4rQPP&E#9;h!6- zxu)VWr$}8D#y2`-tn71?tCikijojrrpY4%UM*nb5hJ6Axk9svFuT~w7g+9#xZ(O}~ zSX5oxJ`7TVG$J4!LpLhTAYGD5O9%)E(k(D_D_znJ(kR`abV`SWFr)!P4h*xu&3!-5 z`+MK-nEyNuJ+^DFeXZ+^>s(zBd=2(XjZdrsZug`Jw0&JM3fT|tSl)4#CxCy;Gr-Yy z)`#%&DVRm2WzgYGsK!o*cNo6-K0sj*aWJECr3{}&!{LhN@G9$&60;3Oa~d{Isx1;8 zwYr3MWQ<$q{k0>;fSM-+7ggLW%)UATgNdT>@A1N`bumLPT+PCizjZ9CIW5-}4?~gm z4{CuZl4TBUQ59m(@{^>*Ue|3oWo3Q-1*h=OkHbHUZG`66rcv4VNcC zo};B&Cb@o6Lus}3D=1U&RKKzfGFH#U} z7k;yBJ358jU{Dm4eTPkb6B|22N8Yk#^OiN9FCN4BRVP>8%@9cxS;tRu))OlQXj&n4 z`iF!EyJPSIbkRAP2rcM#jJ(9ob1kQ%N9XkKoeF-h47w@dEA(zh((1j+jkOLkJpDBG zVbCoEBhsO-NYzLdmu@pRep#oVP)FpU^_~ai8lEBx)1v}+q-TS2um)>Z&_gcsnP;AG zu2JWs+*Oh3ccBQaxj^&Aa<$5s0m^`Sv+I5KNiK(EdD8by>&efL4wppdj2VF+{^oVF zosD=YUl8@s-De+wg^n96FCJa zF3+QfYW~6+aLmI%97(Hykvc zCd|IAT1DuCpWm}9`6ku0s7=vr;01yMDF9!u7yweBTrfR!3~pSQ#2>=@pGpC{6IfygeHbVsCI1S(Jk$T@f(+kovfUv(3FaGW7^~u6*B()xi)x@FP4;udnW%jIZ6wU{+A8)v%Uj4) zq{gy5CH@Lso~V?^47u zP2f4<5&lG2az0CI0kkgblByH?2ipzN9!}d-GRn4sTo?yG88PUQh*?!M6{A{-_pmEc#8va%W zuxOXuy}t%*SlO(1=3S6PYNV6CrOTc?>#5=tH><__Q>q)Ig-l17KcC{19)u3xyog9* zA{BYI`-`rq7x6eNnpvs$N<>Y?=3D=8IJ#FL9y^gZ84orx1#S4s(}#t@gc(*4_Nz@b z_vZ>C5D~Th1n1B{Cnl~^Uh}4C?PW$0*c(FG5>L}nk-rk8>M-orxn*?GOmF!l^w9dO zwA5rTnIvx)QS^-DQ5CmL<>D_7KBOL?b|Z4;OkFUc`otb>DUWZPB2`>1dVx;VS9EVO z6J(40y&ibaHpNbHI2vcXzb`zphUY9Yin91#?H?za9b~JzmhS7@c}mW-WKH zzVZf57D4XRIejaD)(eHT2BG=(SRy-@{oFa4*La7tf1d0tALU-fE4O6B)aJD- z|EWh6$-1n9%-YiW(Ct}ugwfX%JT<$}@VSjfUl$O3p!?6tPmV7@#e-b}Uu#ArDOVOK zG#k7c``!)t(bsyh6Tk1bp|uxBuAlOiWT66JDST8~h9&|3;srJThv^J{`DpKocchi9 zMv?hXS}qcc_z!VklJXi>Clai{RCv_Jg0lHu%`u~%MUr~V(X)I{)s^FUc=ge^UG@%Y zA^m&%{R%wja8M=AYvh0$s5OzS2|~Rb7sQhQSxt^NpIG5q0lw+ke3_T$2Q?m{nA{p93Etu4WPRbbEB(%uQT4-KZF0B( z-@)zMX|-|&FB5jJq#n=%gcx)IikDt$_1ff*zB>eT@^D`KnuoPfLW)*nYBQz%44ABs z7&Ha-ajxgH9!^F|ng|Zpjoj;YZ=NmBnQmoGRQFUKhTujz<&e9yTRB;cQt3>`$NXem zGuFPjMg7GfFuFZOXc*m4$F*z^Icc{EC$H?gdL4yCTyz^HUk$YcDevBKz z%VfwJrFR%j(PE&Y%iwc)?6x4C$D^m~zdW*`2+nr?-k}{WbN;af-dF!xV|o4|6>e7w ztWh}`NCq1HKT^)_knt$LzThUzPo9YO2ns$W=Fsumt~c)qb;;BDX&1iWZ|q+ihA*3m zgT>Rz{R#{3Xl~rhPA)*a=0252dLp7iDt=}*1XZ!ANy=mtxWRS2o?-zG@28_UDp)fU z5gc&m571ZG1Cw;H>e%Aj^*_j~faTer#KYk>v}%7~N6`&`NuawX&KaUY^v1I>;3six zS+nd{1ZS@RLrv1J{&M{y38jS0WlYIn6)9sAVZAz{6Eghb(V!$)*HI&)=Gb4Zmfk4P z?uw7={myva-}lx0_kgV+gA$!W@P04^j>MNxu?m8h`uJ#g%V@JigFiFxaUS@5=?FY~ zqdZMbeb|UOEy1!_$`;8^5gkkqP`Tkb(|S+#y}nXa`pkKpD?*RvS$WepRH)3ytm$ge zcqy~|Z^1qQg?UPN6@cAp>=~Pjvfb$M zz`9%nF>^229!xa2?nSun3UIvvs|GS?e!i!Yvmk<^b$nUBs z@+vIIF%#IRoGVW46CI5=#|rBO(Q}^f-L8GQVVDKD#c#K@8>-Pl<3_$;{=obp%XH3d zIlmsug4K=t4~}_&w{L(cR1=)_9z38*+q_m+Xbq04WgDja3B`N_ie12IoW$<>&z!lf zJR-tVOcTHM z23y_`N2nvI*gv%M`N!<_`QcI#_rt9jua6$mVv70I$bjSN*1*(sdw#J{yxHOa@K& zQ^bz|YYFe0L!zCYS(t=S&WV;na;imuB6Tk7zB}?S7ZNl$1|@67%kg3$E&cMmnR z>`Umx{J)fmGB3uSJ!0e&tJS?w%T4@je%^4Z+%J^3JF zh3@6fZ(Z^`9PiAVF^z825&DIkr_Iypv#e2Rk;u$U;^b+-q4A9+2}KV|EFCqGCWGA87!6kv!aCPNqlH(*!~Hs zqOEW%_=E{mM>AW>o>gtI=)uc7V@QJ`{++}DlLP3UmE%e%sUqA6~;a8(=U$Y zcH-F`Lcu*#xbOR3Y55qQ{&f0_QK^l(>F~Zo0=~gMiEhgANEK-sZ|dB z-$I^gP;dd1nQXq&?O)v`t$(6u#mvcIo>1Ev$le%g3( z2xH|@Bef2^laG8taL^A{>0!_p^^E7$^0|8G?!H++5}DpXl%JoX zpAI&}Jv3x8R4eIQy3bkLZ}8(Z3C@AGAr+!LR{O_3l#FQpGZQzh=zIgeWEIccs#S`5 zbFO+5I^FP-Rc1irz&usqw&c9K9*2&&!IXw&e)x$zE;$VT*3}_awB2NKk)mbADc-vG z_R0K*Td~uedg!?Oz4z7pu{reP@VT88mgys5Hl-e>DhOjAQGXk@TZh+Pc~dL?{+tV$ zc=MRxQez5A6Q}kL?b$1)9eX9p!_U<6cjbf0Ea-$Omg+cVYd(tQbBd0FHl3Sj#xbTN z#k%O5VVE3l2rHZMY21grgb5_ix&038JM;((7T#J1XFw4Wj@4nw7w z8CJ?p2C3UN-*xM!a27GFmIy8`&Te{KT4*zn$TzKpO$JvJyd=eZ94X;4eE6ylq4$HB zO($)A3(uboos3QbJ{qcmun&B|jR-neiApg#I4OT{#CY!onRciyaEFZHsUASct$Op$ zWCAKyR4jR0z%?}Y{45TED$c}$uxe4^6yEu2Q7%9ay?ePZNhtN}ajlfAk}54B-a0$b zmanGF$G4OEW~;6zMf$_9JEjvi${(Ty@Mn~R_~^$*Z=i<^Y)oPVOvJ!ZcsP(=PEp z#jxfiL&O4I&ujSvDy{ctfd=wcQT1-C?t{j&on+C!TzPGeo7OJqdPOXc2-DQi6i+nx zxnOdN_q%(9lPZRQlx_y25F7R$>@P>uoWg#pg%Pf+o-?Aq7WEWq)w-8Oxl} zML3{v;%OX_1R4x17bmFHl16aaeO^>Sh-f^1Pgxh3NvJmF{T1QdwYMKgDo4^MwZUCjBv9@?6weFa1N8 z+9hq4X0)7zQWv+NQ-6KBfz4Q&z8pl3QkzDya3~6jPam^)@L-c)*_BuPbNDR$04~5t zr4sttU)#Y0yR`S21)e_yjm5*HdRd+!G|k0wYYv0QFW%{WvnJaULACZDq`}O(rA%KR zc?tIBVPP)+G>cQg!_2rJW}#Z$=+0sYL~;l8P&}>XoThS=E_V4eh+iOPaYX!TBLeL91KYWL$XwD2B-|5?lGRRV;7Ltoa1tj zI*U>AlLJp*6($!$TNJ(|qHKl8Y6p1kCf&B~p~P+|G6t#T9WWtP`M}%-3zcxIUaZ)l z3)G!QK7fs*Z&bBt-dPO!!N6&3wdlLIoYV7JI6wS~j%LB`^Vtn-3mH!s_mR)!?|y2Y zQ%$jl^LcddowONNlIAr#C!P1Yd_`S^?E7!kwG6sASRWH1XUFz>EUXb;-si~@CBaA} zBpI7fpWkr(7T`T;EQT%)$I;M{``?Ei&TedEeXzyQLW@l3qKi@E#~fc3K6F>W1bM0N zp&{J=wl!-`C-F8$YCF5OjfLCCPKwb;{WohrzZcfvt?B)ltrq1tpn)7$3PW+{#N`iQ z6+kI^eIKBZD8yc)ma=q-!dR6GDevt!eAU$?WWJSk3xb%bs_&QFKH}i|?Z-)uE5`_* zVUH__D@89^Lfgi@;v#}JF{Q*3ZoACtCaW~#KP-i*pY&a|SFXS)XqV6Mu8 z9ZhReM^jsH{u~`)xQLouDIf2=iL;@O+biZK`1YkOP1=_E2N=UC%?b}UYI^K@tmw@wX0( ze-@)zKQ*$L?q?!Hw9^n!IJBdFVxCq9FP?)~ETq+iD33IvKOxv8vA$5j@+@6_be?~b zfJbbXA}#(dTfUJj9r>N5{ou4I%++R|FFuhgB0h<7T4^NYcZ#9`CO(P?v)eb|qCApk zfqTgU7q~x7O6KIt%65C^{|n%+LK+qswteLnR|&Z!d^31_f&@Z=?b$dpQYg?3x2jU zMGq6vK4&;J?^C0|f32}_k`STWwPp#WgW>sU-d1vZ4~rl{uG6u^+j?PpfU;7(j@|uo zwlnJ336{Gj5f+f?Y&%s@BWhl|EI%&>kwgmYA;CgA079P!>s^L)PkioJTgkp620v1U z;IG+F`hD4S*))ul**_#Tri+QVQ9+#P&Lt{19l5qw))JLe?{7*6$y?nsYU%As76f}J zc_C8Wgs^?N+}rLo0rEoW(N&!&oFwvX87g(rg4$B7;%`29v5nh55J=#L6ns#<3OVq+ zStayD(OLoPS@$d;2fRNF z=20KC71VQ)E&13htIbEtOxphC|EXY}smi~iDZ-K~2;*olJRi`bq^eOzI2vICYM9?S zHTMm^?!D2*Vbj)n_NTVvZK`UTfRtxTBI_~Z*Q8TXi-+R0iI1B47-60zhG%U5;JAON zth>*olVX2|{;FD=g}zF4J}|FQ7-wqrBi0N$)E=9r0E*OcIxYs%TSw`XB+jW`8wkbjP&jG6r z!w0<{-7gJX4v8az(@4|QYTHh`5nBte@W>ooWg-VMO#`f#N+fw;*L~X){--G4R=g_L zGk2B>HJ_i6+qW#OLbK-0wPOa1H#%dtv}_bBUTTvPLLrq^x8igEq9L3$KMiYyd9k#2G0jiH*lgdnYpv zejpnh6Ks+xW`#_xXEqKj*2$RU>E+w}Lr288f8?D>UIQYuc&5I^hNlG1;Qp<(<(rO> zD!qS_jJkzr5s3RB)gwB{7+?Nx=}3Rf9iQ&EPXdJn-d#urw{sqQINwzFW2ps@KS`Zj zJ@NhFXBBgwvIeXbZml=B=O>5lx%4HfE* zvEsG=A;CSezwzkvW)tDP|L~e>Nm8de{JXDsG$HF%METspK%y~z=mIhQe|hj`2|QI< z$*YECHn8t-<`mXN%aT0!$OU`c;q!I)SUKt+ZuWMStE{jjo@Zf+ah^GAaYl|{v)X+d z*Pn!@l`Kw7+8>UG#;1YB?Th+tlM-dFdt2ctaNDxbaG zDXH@Ek0UuGcKwNQS0eAIXwv&Qknb@^Es*+zFVqtf~8)8y;TbcFmWq1NwkC9?Ygo6e+lh10zV=iSBPqhjBn9 zA>;EJ#^-H-k3}t| zpCY(lT}p=O_3oZYu8~X>9nI{281T@o&f8sVn2%5cx#M_t3=UA| zi;hxPVC90n$0oxG!780z=+OHSl}ZPT3;*|20Z;MmAR87&>2sskZyqIboCz=&{oMM! zXFl4uff~yugv9I152p;gsq4~Bd*FFEKF&CE){{3y1F7Py_}u=rn@i|+6TU|oO*e`@ zi#~+tbF|TYnI7${jt#@9e5`AEO=7PYO_R_$&V2XxU$sNJ7i#lypT0F2NZ{8(q7$!m z{S1n~*0QKFrsRS};~5BbhG9byDNx-giTV|BiQ_lo9}x?7L&wUe({{{*=3$Y5Uv-f; zPA`ZrkMdiPt3{8(vwSIV3`<@$lXM5Z+tgfUgaXtEX1K^A_1DrQztx&IdWP z+Lm6DiU5zeZxe@#)fDhJM0XYN=aF{YMYJvi8eJCNJQCP9_sORyNZhlQnp2haVRN(KysbPRwn3ouG$>B*0Ln88 zUUa7SzbW&Tk(B>SnRlGA!{bSpQ${6#A!L~^KUm`CjAbWK{S-019}^^ogP2#Qh#Z!s zqui@*JeBTL?;q1Z#F&~M>Wh3Y-sWnRA0WY@XoIl!+;oz~{7ASf(*$O< zcwK~NUC#yFTc(7z0A^=@Z#2EtJhd=K`bG~&G`JjW31H>&9&gB0t8QW$6FV^tL0Hqa<&s|2HWw*rWp} zf(pn@skCcGWz$+dZ0C|=b^9`J3&6dav`|!i!MHSN}|bH0Ar_W?i7sg+U$}m*_4coFat)(eDL_W~{jf2pSpXhu2K3 zZ=1c}aY^N1r!^p(wjdoDYe7TlwA7cNjLdd1-drWRa~b z2~u{~>SD}f>*9BrH1w$~Dk3jVr!& zes_B}7I)y1Ni7iU&dQ7l)z&!eeh@`ka!C+Ui;F>2^1jx8S+rzjWoO4b<(?O4K{NGX z7~)p%TR+|QFnW-i@7mR$*tp~xoa)7Z({7*(E|AlnbVekqK#IwZ&oY2$bI3g7+B{TKEZa;0ct+*0u26Zd*C?7|4qU9r$|H;I!g_%$1(Sa4b@4i`493DP(HVY0pc z_&Pr)VWk2_Kv4lRNy(1A^6oAy=2|=&*q|YNyHoJ=`L3g2j;#JzaKCcs99=ZTXrvQY zc|dt3^|!V)_8yP7m2N;LL0w#n`8h?H+G4}q2RYEare}ri2Vz>h#rJ=wV|+$c4??|L zXKE>o`Hz2+1RCbT_Fq(IPwlUq)?i<-1r(#p7=s{%|rHxdu3i3LjK#j#auRp_2i{ zp|-h@^=zoX|@98xULj=5mr=>tGcT{OlhC2Kv)%=f~Pj|o(DEIwLjaL5!KGu#;08zkl|mN3En^V5`~8m=^8FiS)E|2c%*{ETU2~y@q2oMXa>Sq2IoQ{G z`4)4ZjS;oy-Wv|A#wzU$V)xG;9>kmJ8#aDb9i#phD?;NtUU8)j5fQ+_WX##-3Z^va z)95;ok`3vfd+!nA@vnltNl{T;<*w(#=BGDJ8dH?d!@_)RgSSD0&4y`jt(@Rut z$Otm}T?7*$pHj69$FZn9*{+|^FwtWQA?ixF)Ee=E`^trbddzYEKmPyB$$a(rh z)Xf35xu}f8v0tLcKNYIW6vitVM=ZnOAf(j*QlNaD+DdBhH_1WLgNCK9RFNi_KK8V=8 z^U$ejE$R>V4Kgk%%b7E8`JI)KSbe^W9pW@CZ_D|B(RSvtw#m*bn92G3J%R<1hmVek zvN2oHE4C?OS}QqQL3FZ*yp&DW468)NO)=Cxp03X^RPz1&)CnN9l-0h<-->jxH)GPl z;p0TibAMT~mL)lqyz$Ho1^jS5@M&brfPNKi?e%ek!yxflfb{FRoKgdWr`)YEXbsvB z%%&UbKOVTh&HD3QA}>h}m?jzeYWU^4gwnfBJ?g4?uT90+@ z(>AX}z~0mh=G9;lyJSU)P@D>G9RT`+yf7(L`X$M0l`HkD!>`v5K7bwIFpLm1QtP4} z8&O7Sf23=xMZt@93|rNC)cjZ2?x^ylQhkv_CR3|9fkeVR@Fd^GnWqh|xbmLpu2R@V zQQ+^;T(?RB3`RPAEjwk;dOn)3dxrQ;J+=09Q&JtX54R99y)K#YBUlpJuS{N3DQrfs zGQY@ll|?6&fo=9~N$QWO#6_+=BgCcoe}SGZ+Ryj2dlAOGg*F&P>Jq6i%7s zdj@N-qZ=G{iO-%n-niz~_~IzWVb@Y@`$UqkNZRi1sWg)a**uPU$i*%tKlHFubIP92 zJm4#zsGjj-1c$k=nmUV(*mNz%g4x;A@TZG~?p?>}r+KJ6gjR<|mkQY2nSTJsssuIs zOjSS(x$bTP>{2y*{Ub{CF9ns?{r@B>FEgz|&|8B~Ar9OxFl=oIv?;XzT;lG5E}(=o zNG~s9P`%G*yMt@l-}=hE*%{B2zLs9%llf4;%Fh?9y`_y9@pd6`6HM_z;6=qLgQuTr zi5NKh!xEK3E$I;Uw$@rzsU<Kcz?`qlYw9eWA@CDjatTa7bj8!) zCG$?g3o?Ff)9)b4AL&j3yE!BBJ_2#{yI=P~vW^TCeDki%54v|7(39~_;IfVckVqS! z|G4&bULm#`QZV|z`WBBn`kQG*%#_6AA*3x6%kDZMPaJ39tDxx2H=ecl92V5JL7ulc z5(-Nf8~owJ7Mx;ieR!cyn(?jRHUEp*=r z?ypPFw3Q{(4+E*M3tCcO(g8v7bg(8}GzTc)Lx6DVCk(piyI2)~iDE_3^y_ihf5kMH z3b{Dg|B=$-;&O;7zrH!HRyfXY*+Hp`sobp4`F|LP;zlHGM4ta)_XrJ|j?P&ub@K-U z9)?GoA<4LfGn8b9zeppVxcMRl#)ZD8~`K z^NUB_h5e?skCtGFrQOCtwh2z;<(a`jTv97jsFG|AeD5B&{mox$oG(t_oUh6;FeO!{ zv(dzM-)O`e%~)H#g~q2F>;i|pL`I#@)tbr!j(H@Cwl7t;{w!DWU!09l^uc2^`Ms-? zZx3<&((r%9w$bD6k>FVvE|%v3Oy67=kh$1lbbOnam&cbg-Ka&;#~8!am_fg`PCj{IHIY?zBev}=%Kb-e5Js)EU4NF8Z6k<;Z%`oU^ZHmqTbm7$ukMK?EU z>0U;Qi8`;?+t3E(IyZNh%69@u;RQL}pM-{-SfL2_pDJs5=oszPt{4ll;@p%i-gWAOQ(znlajt0{yC3!!VoH{Ec= zi*hb=k0Y=owq;LMi_Xj>!$0P=bZ8vPng%ipKF$t5UEd3OpwDaQwOHNrIS2TBQ|LGY zJBNMmF0nuW;>AGmKOF<(Fzjle+ve@unh;d)&%2}JhQACGoSVvGY8l|a8Vf&fE41DW zBZNpO3m(J#jjLH;Li%Rb9ezN4b z`bxh=McQ9i0NbnKn#XsG3}xLd_O^XmX@8lK76Jd#GDkyvBkvt<&awA9jf6_O7T;*F z3!MI^;22FIge<)DRm|M}>PA0-L!YgB!>IJ^tX#YE2U4IsSO%kiI(DAnq4dYs6TE13 z)J)zj&2oFdMue@`T$z7>?mZVr=kni#(sFo@*((nJo5=9fUvmG+cPmHER-1VJlvxlq zoWJ~T4sv+w1Bk(hq|MqLr3a5@o{_q6V}nc^NcIw--9t9|>5MWmc$Asm|?5L+6F=y~# zq-IH#Mn!+*ar#(F_r>QEcn14$@@mfqvYW`tIBX7ra9otDzu$disHQ3+o2E(gd~HC)ssg%m}Z0oQ7zT5>)cC$Ysrs zWOz3^locLLMdew&5Jr4r%C%rFqZfFR+&+R%R=4r9QB&WZs5+nFxP3x4P)T{fmZJlP zwn%oKu{Y6HD5moIE$>;sfROZe^v2FgrCQ_Wx)86k)+w~?9jA2*7bK2jR~`m4RzxIa zR=E96O&_$Zag?a$i4AExYhvqjaeF-ji-a|HZy8;cc(K#Lo}9p%^12=b*a93k2KBht z96)H;XLCRmiC0|)euS{A)4cz!wc;P#PyYF>MDZm_N7-tHiL>Uch*}aT3?ul9+b1$s| zZd9Tv~p{`BnFSj@h1bhjFcuB4#O9?k|{&IVEJGFsHIT;Ruh$SUpR|2rTR=oM4 zM%y|mgai{4z|4g8fyG-%s}P6BU%%@hwvPItF0lVv`b8+^t2p7#0~ALM9_V)g`6fPhgs!Ap{L^Ojeh&Y`gq`y&^vJg~uCz7VUgsY=?8I5J??L1{%fca{dyb*40Ted!4b zIFyjt>Yu@C?ItGc1WuQZq*B=8CGXgz85oYXIbPi`U^*V3{>11Y37n}(JoWL>PyH*B zN;xRbW)9N)^=C?8qks5*Uhi&t@~P+O7%{>%l1I`ml<@w5!VEIL8R%UV+ZXu9C6$x4 zDs`n`?rK=@)2{o8k@b=j0B<8a{Iu+Dz0egDj zQR}M%#avG}oAtd>3c>LX?3wXiCFc3dMZf+nM^!e1Vj^$DaiX8CW*UeH8^P4qe_!Wd z;KGyDb7%>?5?(AC6q4P%L&2>g+EgM)oU5mbA8hp`CXd=^QoN#<)*;g#sZU-m6@EGL zH_*K|vgJVVz-XZrIEAQ(;1=#7+G^h;v7JS||lJC`l+kwDc(^M!* zMf`sy0X&r`ZYeleiCy&rgFe6^?7p)eOHQ)Vs0EH4d%KUeXw;$UZ{TuY90_=w0TL$8 z)MlqXuOqAkGryTI8sz)x!zgKZ)b8sTL?x(S#CnHPWuFLPZGGR0FX-1o9+z>FTR29E z&;-R?k(u?jYkmDt>Cf0z(PpuJ8F|6T6X_vV`5~JVPpc}${OU-<_h%{`@!lZapJDir zLk+3U_>sT%4dw8HnGyHyXr0@0^`A9ehAXZz9Yu5tmbFzH`y}psHp4uG>|fTk@k&2c zv>#&DZBS!j!>^mNg?{D@191;;iksG4GI(w-T=R6m;JI0IY8*LE@0@wbK!vMA;ZN36 z$lpPR_c53DKFdV$P`eaGM~YWb6fre8PhMs6czr<)&~AL?QTqUz>%bG&%ro4%bJJo~ ztbA)MA0V!@Du#MLO>;I5C8T{%BhZl#3%E*sN%kOMAN#0DIs><-@a{--AzbbifIAvzH|PgM45{QopHD z1>Hy4YeVCqeu_|77|2^+O^Sa48mmQj@I9==ULD2ZzA(S%xQ^*BW?;=J{wLU1*SMsa zlC&2)cTz>dciC}nLWcbOZRD4xu=+l>B*B#?0>O_3ecJnH`lyx>bK{tglOwtaDq!mn zJ|&d1ASC&Mn9+zEb}~Q@k2g2u)spEhZ=Z;byUPPN@Ms^uhnGG|h5lu|RPt8Lu@gB2 zpMf_+J1bGz_GKPPhubO3MP9DT67z`=zJhkU!RJT^b1~h(9<9K`cK8%1+JGuBIA`J1iI1#g_;dYk?|E3>(<;>5!kv}ozVo~&yXoI6 z-{P?erLb&6>HoDZ5Vc$@Da1OhznGpgvDEo_8tEEK332SqOf1;`$RhwMn)lu-0VMb^lxcN`$G4#(d$0%bSIvJIxT)7ATZx#S54<#yto;fWV0|nCbCBGujkx$s$ zCm@~d+kZ4q%b3cTM!FC0Z1Q8D?7o$^#d8p4<~rXmivrq>ei$p?&T)?gx1A~JK}9%} z$S1Metv7c$jZ8m?u#Or%ZRJkwH#@9u$z`Xl2}$_061x0)ZDd_BW0R-=Bi!*7Qr)<# zH-=kHybrDHZ!|dP_nY!rhccXB|b=X%VI(^|UC+uXo z)y-d(;tew&FFfSO_W3)KJ&DUm?Es}{LP%u%o{RYn7^3-7@*sx1v4}Nt$IiUXDU?I@ z39c#aXNlqq@0uOEb)pe=vH$rKQG@0)C7*4-4XZ(+6pb{8nydUKyKi2f7PS;(EXZIu z-DGGYAL1kPCi9LiV!#H-k{@6P$bo2>`hn2i^YJnI~OFhsDE4TbQPbdM7lym(v5Hosxkf#=S6pmc2_B_lh@?4 zG^|1W5dq-T)b};u=6e*L48H>+IXC}SR2=vg21S2yQ@<*;ap^T|D5S86w=WqW%fWcJ zUsHi-{_=5KeKMDKmuEcWXW+>MJ-p$hbX2ukVzYz%=B?_r&_LBdH`oqP9&a~)IUC?n zQ(G#cgtUu~y}*SmjMLu_(0iwnH*X*I{KBNTd&uE;J1dT)#5zV@_%VFxBVwE+rSxaZ z*^U@EB5|8SprJW@R~dG-L4)~qy{Qm5yMjf@6zO1oaTmE17&RY-MZ>5G3M5$QMyNzl@Qc?^GllIxN9t8eiL(!jmdsCg@d#vdrsqNR7?8cg2x#DOM$P3bgg95x zm>aU*TQhYXdD1@OFajf&3V|^bp%NJ|--LO$EK8vTDrG*4(Cu9Pa62j|zIXxYD6|gL zJa?-0D-f#D`VI9J^_!2hEBGz>vF!{RE^B^&S9#;97D9agYM;kL{d?J87UQ<_-z+Te z?Ij-~mEePrAE!7b&RsP#LtB0u4z+EjxD8w+YH_wY<_&)n%Ib`}LROW0?adv4r3u&) z`WH+u5M<1tg+ky;g95P|s`hH3wH)A8y*pvQa{d1dHMmkb4<2|Z;V)phXH-t~WzBj8 zoX$+~O?LdxH^f2H=kZCi?o-mVy~!riX^NJ*meLcyz1xZ0Z|?#zrnd0g={*q>j+$B; zR=;P}U3ZKiEc6rsHjYY8$V4SenOe-|IG%~=qeS9={4t-6-~!cqS7v-~CCh&cLjf4$ zDGiJ1pr^*lTQr!XJ^I)G%-Z+(@!QXtSyLkf;6#_bvhPtz3F4@kY6R8c)H}_c-0gPb z---acQFH&}@>)EZ8cNOum_;cnvfIV1^~{?%a4N5Z z0o&En=9|60j>+HGTAC5p_POxV3;@NN_Y2WRPq<58pVXUC%JAAkYVE`32?_#zZN$>t_$iE2L z!jEhP)f3w0zr3(bK;pSsvXD*LM3q?niCW<1$0fu%%wHM#qc(>Z(1=Jq@ytIYS0Z^k zotfl$nIjpVCtKkB&C+C|yT!ijyK>7!xXtE=HdpuK+RzZsu7T&i2TQx7%r$uW-2tdJ zp3#XMiC%rmV$_`0^LG8f#m?zO&5PH2wUf=0qx_w+bIVDrin15>cj@upC7%NNqs^@i zKeyCDmfghpL4foTNHo{%XyU)5+oRpx(7yhy00iet&i5+ha8CP`Br{9v{l>!52Jz+OV zgphS;$MdeEvEhbyApict7|godDf3E$0`KrVo1)Xc50|^`E$Sk_IqZzq!W_G#N?&jR z8(mg#yvX!X!b+mk!I4miS>sL3U~~y$wf(6{JPCEGRsFIo0|W7t`n#qf%S0q z^W^YtsPpr&!P@puzqaF&y2@m20?Olpd?ajlsflnEboOK;sBuIb0SOyxl6k5&I6Rc6 z91MlM0)3XCNn04a{I3JI{kI&jwbRYp{ZPChN@xhY=iq0a zf!UTWO2hKtobg{q`z%QbSb5vMd+P9W6EGo}N6n+1#g>NLyX_3ntaoyk#>wUb(}lZt z(@>Pof- z#i}J^Ngj<~+G?t5A-~~}n`J1I4!ChbMi;$C)r6}7yvDg5stfvgBuiCsf{p@sm)A{} zcN%cBpA2uW<=ZzqDf}#Nbi`Vp(LpWI3ggL#Uqk5M-M3P`VS2PU;*kDkRSV#qh}`l6U)@$S|}+yLHlbD zLd$8%sHW5Idt`^(4i5FQBkCT}nhfGru)q^a*<0uEI3)W2sCw(5DC4&8Te_rMy1OJ4 zC8WEhLApU2C09ziTR^%?N}8p+BqgL9k%rxM*?ms0>%HISer9L*V`p}TInQ4l@%es_ z&m@T>R%dUw+I;c?+wqDM%~xLgL1KUE4c=bX6rEdeTnRcJ|M7ONkCqfw4|h91EjmB# zDK?h~Safjh5wGtl(Oa_#_?3~BWJcI+{{I|$MYovrRNid>NeeXb)L;uD#sT52oBy!) zaHtlQYWqJ|xt8Lg5u%f=xrzUc#;L5&^O4`G=YYxpoC%z94Hiy+dgv--b+)PQZq8=! zNxX*gA(5aHWoVgZJ9HeQ+BAnv@-|+fFk{gM zrlTFnkByBWDuUdc9X&R~s_ewiDcBxBIc{j~kP!dEh=f z4018Eoc@YY9bWbcUuhbUI6z@cF1X^}a=c!vPUD84ck#Avp8yb)ystFhqs@Ij_4D>na+$$J zl#Kri5socsum59Q42WwB{)su-b|8a<9XG<>07Ybi&(=916Dx_#^f4XU%#|)b3zaSs zRyEQYm}2!&O)9_ss7~&-*Mt((H4XLxgbN=4t)hlej$c3VRkn?XU*faIdtuxk>@6s5 z$Yg``Y!sVa!rDhKd8d4v7EYR1fNP?6TW23FY_!GqnEC87P2Cf%9Fu-996^>SBg3!} z5y8Uz`h#z`3{*{RO-xu=|4)eKQAE&5Ha_G2s5nV5w2}Pt9Sze*^f%^a)EYr<<7`i4 z4*Hzsi|el^+hs4l^(32!`Xq1qb9zvm#j>dA`IzE*Oqljb`;EVauD^oZ&Gxu7!QcE6 zdO;f1`x(VJCpl4RX44#PtMHN4S#n3>x1kssTyGS(#pPnS3&S4F0y7N2Q~K--ZzY~n zFWMPZQHaCnB8~J5Qk4D81cLX6+UKNMKH2bWJ!<{By@&%mB!FtBEhFGzVf&L=6LMy{ z+EvUqV}*`%5JvvcahC@ow8E_EWMDMQKlXf1UKbG-N(-u0d3p=L{9f=-o)IAg|5?;J zRNNq4qJGd!n&%g>TntQ3zO$wB-&WaA>~T9wAloe`l>~S}+Dmz$IQDn0AEx z6`R`s^heNqO=!;~@n9B@@Kfv;0I^s;a$|)b&hfY_2RCT?+`K$P7h_hPW56#;)$6MY z`T1wt?fR-jOFo69EokTR$R}$SH}!)HFdJ(#us!H5vh2?xwd%#B;rZJS22oEZD|(;X zRceI2mCypj{y{>o=9YR*I(EbCjX$R-BDrepnhW*0!S!_! z_)6(T2i3={uQ7IAIHux<(-;LsUGT@y;&TtcbOTq2*8copHNQRGK1{l}{KZBL>raHq z2j2i?c=hZ zFaH74XX>vAQ;4tG4+z_pq3g#?{}oug@dn&(!(WO2FWZ3yZm)ROUC@#1 z6VnICG7Zvl{Ul7LXE~tl3SJXycGmdM5@Qv9Wq7tilN7Ix6K9B4@0a$IN>psf?qdcz!KZ=XTQV z6}<)VQG)yP3^1dLPs|J@)D(H*(Scm|yt|l))%sPbQBL-S5-`Z5c`323ACKTwclld_ zsC!&=B63%rzddL8RYGZq?7e_jU4wshbp5tIh(4|GHuZfUW6&|C9QGqj<>f(M;(b@| zNY&|ZGXVn7Y_TF$^8X3EC0@J(m3`kl7Bqydv}*@Dy3`m>X1k8-X}!d{kjb+-|A1-! zh5z6ilEq^>@O8RYnMlTbaV{V-$e!Ws9~+8I#i8qc-Tycn>+Svdyy9BoYN>&+3u>xj?(}T0^D)! zZVY4oocfm`sQCBa!CB~XZZ^nK^Pk{vtAztlzAV$+s-A5f?_-Nq8~j@L1?sw4LW;ns zvrw!B*F7#4_5rO0$ZI1V5~$xr^`akw{eaF==OPBA^o`DPsHa|-Ya=S-WGdKik3`)~ zZ@T`u!!&EVffcGQyJuf=ud00VV0v7Bz?B;zkFZf)|3u6*FA`Y39-3nf!8;!QmmR}E z#pQ*oC&?!7dxH!uH@6LsL5#ONZoaqzAr(u8^lLI*zoKPWM=-KVo-*TLjh@C@nbW`t zAO3J7q2vn(3g$8r6*qiiHUo+zRzs#?#qo6m(v!&YR>_kpcc2AF?53rA1zfa~rtXgR z3Sk2)5{&k&0aT0|a|kQ#ESZGCt>F)e&%`c2oEV5Ne|n0h34+f2Pyk-Q-l$f3RnC7- zwu`)av}YnZQ6e`#h%`(Ukj5bg(oE7=ud0y>}4V|~=0vIV-Q6v;<*d>2+dwPnV<-svuxk&|%vsd&>+ z3O?5WQ)-p|F-WR7#lVq+*6Z z(HQ1lprp3d@0W-viTzC&-tBJ!lfmnJMyQMF5g0dOC}@8IT~PAL=5a{J{;*=+G=TA?HCdQjHfXwiJ#4v)^|IsM^lf#HG!?e7)Vj}2ZMd!UyzLFlVOITSW zXZ}gFYdKeI>RxwVP;D|h7 zwnP@SAFu|AlHdVuC&QKfCEr5IutOfPc;69QQYq+Aa^R=Q8ro-3j-zYujIq%_f4jC| z)XSFQm7k`+dxR_)JV>s;$^W`6VsP+*N9L`4ixJa6}}=jEZIbZf9l!`Vzj{iGbOQ=QyDvuoE~>SDx5?d5m_ISDqoaV7(* zs#+Rg%0fLI?cv%Qg}&%lXVI=NG>+)uXxjwbv@Y}9FH;G2X|x{`LzWMFA`iO2Z|VOk zfPwojF+aITzP;;DHII*^VASQrr?INuYH!yr-R922#n0DHzU=jZ+}lp2psefztOU`| z>i_9!#BdB@bAeu+tmB-fg9lpv`tb$D5>4X?FYm zwxM2t?Cr1&gRwb2dp1uY!Ld5-fX#Em6$3{P+n`%q+}G&DD6MczttNZ2zPEjr&*D$Z z%EYL)u@XTwr{xlR+_jloCjuY^K>qH;+*PVi3bNZO@-n-&*G)4sVff6@)?4$U)%7St zKG2!Y+>itJ?c#{*V#xCrCPh*Q0>@Y1zrE!bcQ1zb3z%v5f@~Y7oG0<|JWlhTp;PPB zOkUuw%fp2`j^4~y_$rRe&>gDgyf_tHLBx|RO;g3)N42omf=sr)yE80w!P+;MMn6vq zxfElexP3Cs_$JeGQSBJ`w|%qq*$nXQ-%sQPGN>9NaxwK-nW6j7s8!1WZU|W0$MC&< z@PFXq7rj@2@cBO+0oZ4A_T>0=-EFeyr@=y0*=Yzv3s(kI)!EPT`R5k$sj5 z%8QGMg*EscD&8b5O1GLQg$bNoEZ(-Q-dxm;fZRE$XMP)~^)4X@{M_X6oPg@?-jX)x zHkEOhn|j^ZnU;@1^NSt2hsisb4XRVh?-)s|C5Y8zgoP^Ii>d9J80MtRwzx_s(9pGz zBBb7n+8OkBJo1`>Uiwen%ldce^wT;|nUaEz&z}6<=^P{ssaRTQvXy>xt>x_h`sn$T&HIK# z>{a_BjS5*i49DlLsOHgpYAw2ziO%DKNZV*Q;Z~}+hi-k9YH$EU~c$7~_Og`E9FC)}{5uA(%C6vUP~641CJ zOQjpFGmv!r{{%^;aLJN8fD0O_0dXL-5FaAuQ0g`GfHm8q%*y?ORkFd<9aWs1)R`e^ zZSOuxVC^Ri9ElI!CT}^xIbtj%tW6o(A-m`K$?Z6pP#2^>lDjSPCK!;HCITWo0I}?+ z+HZL^T3LC1aH9y>a)kjlh&YD|cM9jp=R=@Xq;>V0wEg?LAFCn=R^T!eg=s=b(X1!7 z=1L+HG*&=*2v-Z}p!1*q$BkvvbX2-P$Lj|FKrhgp7mtI2Y$kpma4TYabhCiHUf)mh zaY|YLLS3>nA%>-&i}eJJW9;WKpr}!&PtP2$B6J)!b*TDwkUU4HxtWeDMaTx@-)mAIt)6~baOp87tpQM>8M^U^O4wvMt( zr_}-=Odp%`7wH2V=q#LTp^Ss7Q{1%iJq--z{DD$0^$__49>MyGsnqeTsONnBDT(SY zN*;BLT87CDpByOlR7G!8BjdlLNYs?AFUT+1T1Ctp`y-j*xg$4vUiSkci z_XU}Te+j&v;Xi|5&Y(cny}x(gikcr!7Lo%nx8wSJDF9W>t*9y7I{Dn%Uv62B070LW zt&kLe9R1H7SP%?zNIRFqaBf8`SvDbOdk{xWG5&5n{@XjF6Xqs>Ts#g{p2=jqr;y>J zbWG7c1t3!;it704bkm@?t^%lRS7yM^5>t^{XvlT#Ci^cd_8#HUzG1Xym;a@EUD4)B zE!5X}AHS0VR7he=bL8oUAs za)J|&-hUy->CCuJyZFkD{9T@JhqZ!4jbNme`|C8O2n;UK4^({77XP2g881sB+|-s@ zN-*m~99y0V2LXUr%OF-tezP}KCqx1SaE^N?*)e*@GP}}pMPbBu?fs@cAPIu9xPC)YC5A1I+HTrI;>EOAXR}x(;^9> z^#Hd3+KGX;wmj?+f~2I4I|~DAS?!1Kx0odU08&(PWMpb(j*o_pG$iM^qcoclvo_48 zTPM#N<%dlqjZO51;B%+-OtM+Z@y&S6T>u3#s4%iuwD^|K>RX&BK!Ko3~LNudohKn!_<3iq{!>KN_|y zBDP+-W|x0xTZ{l`WZCRr1P5JF`W#GQbuUI+SOI5;_PY^|n*og2b4^G_4tXL5XPG5l8=NVz-k!Cxqe}H=roN&O9rT#lF zpXWjH)^YX2#nGcpZop5v?#gEThfN>3jy0yqd4${`aww5W$s@&Asv zV{Wq>UP+VDy30!*mwvit&0AJ8Tfp9k_QpMKB%9Y@P^HE%YuU-nsL$OC+9cdjy@^4N3#95?Tt(iW7YBFiBGB#e0igqT^=nv$>JS zZ-at-j#fDQo|Bqq)}NX=t+e;ly=%S6Yz zdo`7AN$S__ikJDL+y0xLDAoS<$)oh^<6vM9-*VBYmbbc{bk@9_(g$OH%g3|+WPiNx zCX1{%1(C|hy|h4CS=(#8P(MPtf?s)|A!ZSj)~N`dV3EWm;(BN*(q=>cx!}Lm?m=nWuSOf1J|@flFY6$4<@dIfpD~6B9cl)7 z_uo4`$$9&(H*=giwdoL0Lq3_o*>Ms6IO!cXVe!tuyyd~dsNSQmN@bem=+y>%Tj z)W@o8niVz>eUZe)D~jh2&zXmH?-Zzweaam=V3KJEGlVml7KqEd_j1Nu97YPo2|VAy zLto-!BpSkrrgQ~X|lF^W5GphCM13;qq=gdu{;_P@S)Ob&kI>Y^nkqNM^ ze0hr;1Wj!J?~z*u{Mtv;v`-J`Q^-}szxi9BT~OX0pvVS3z!dt6eQ+!Mh(^4^VdC*s zNogHt)#+PffST0W?5+qpUH%!c$M;K~ApPChN)?D2tqN3*_(iRXsn=kcw4>hgeCo3K z|4z~y^8Y&RY=YG~e&PN4)#rE>Qp?2pL$s#ikL8gO4|XV($_FY&e}jAe5!9F?!u;aj zzXg2ubShe+J^8j9SsYaswopZbJsdp7tQeaSf>mow0Z)| zH?C}i4+MfeI)CeqG~@%+jQT5#Ps7p!?sMnW;Zhx%s{dyp|ox^b|zRt&V@ z79Q|Hswhtg@(r>PiK7u$s|z!Yfxj4#9*|b(iUy6tDS@N?7HRWH4Yk+@OsCR{0zIvy zVpBg1T=m5ackC0=5q8ghb?xcbiU~?@*YErpxm++8TW?VeC%=;31;rvKgArMABmx`b zEb!>JdGGTqqwmL;hrg-v&Y_}wU@o6n_l4v7y--ulH;!9qLFzHzmj`!iJ&-WQK zgnh^Kfq5)J*-}|oX9+?L)BZ(*+Z^v`!0)*}`yw*q5|I^Tx#s}BB2T|}E4p*`*VDxm zi?F16{Z-`fQUfr!dlBp8s*iOeqwYn&5{|&L!BQkQOHWhy;%q@U2&LUA2B-Gy-Q)P# zcs*~-EP;vBot{%+3T+?-16u}!b9^-CEc1MQ`-p8Te+$>Lh6RD=3jC?#g(W>E?J-4H(@FdKqgf9I&aoKxpy{^BZdxngk6^tR7k#TVMyIs)Dw^ssO{NE@og4YqE zno_E%tt{}0XK|^)Byp+E1o&ez9x-qZkAuEO44xxrFg8^9dW(D(Buv4E>i33xH2Km* zYi){xfs!~Mz06MCTOmdbG#7Y!F5p8a96>nU7XvrlXf)qZ^lcHv-KA5Ki+uXoDT~0HX1i^77ceb z-<8XF>M$Ec3F%l2X52gJT;PqQ*b~b*Sw`=^5Du@f<7U_iCAG&m_3TG&-fAvLbB)7R zoy69G^IJWdh}4+thW*oJ{qC5^3t&k)@LH}J*UT;)5OzCY3&*Tkfi&R4CP%I4T*ud_Ho3J6l@Z0xr5T!KZt8BrB0>wqM&by0r0{8*B6*8_SUFUfM?{4uxQnP!J$l@U*LJofQUrGJG#>fzGrs*_u}6- z-;Ncd|8$E~P~p#H8%TM>%A|ptq#ue@dxhYMMOZ)j-V%+14sMXqAmC{NjYg>SHOOKI zI`x%T^bO52M`%#XCGj)pc~%P|0a(pPpdiEMR9RkC1W;bmYFAJ(viJDk2?!iCQaI}-3@wd|YqIXGT$Y8Yf81n}Xa?2*7?XwY+qld%S+f4eIW z)$FmNT_7bKn2Wr+v3NYbz9LMLzB3h2>&c4r<|nIrq6CvaZ_*BnoHPIjA_fnAFs>1nGJ1T!)Q{=qjNbjVaeMoDR{bL}@$JOrhXGte^~yz1-v8JZWMmjmHc z@f2^iUm($^F7){U6VO47-74UrgO4SGe>BYQIkjWm$bD9)FN=AkEZl{JV~e+Q62G@K zN6C@>R6>?isvOpAlfm9CK+DCz_NP#UOLDJvG3^>HRky=c7(%%zXC)Cx&oPugjvt@s=F5MY@D-P-A5b=&Boq;k1 z@%y2bCD#t2bG&v%xWl-Uq=a4MSk98Y)0v{eHIvDpL@a36-*1Wzt+ZMzj+Qi3(&VZ6yTGV z=0jj4$8D5C4C|HrqK|A9OTw*_P`}eH^Tk>Ki}ivNrPY}-qsR$yOI+bh5U1cPTX_?i zFK>So>ZZjGYMeMakWkQfGip)ob`xD!4c&Y>b`?j2Gpy0^k@Y(lG zOP$?{%}*Xpwy0N`n26K$>k*_8DC{)>wj^GP?q(^SW5>pF0a4pAV;LsM?0A&=O<{KY z({I|vFciSOdNM{m=OeEFU^Dn*>?fDdaoZIp9h2C(2o$y<8gy53rM22()Zjq3B2A?E zJ*<^Q_6{2zmX;x(l6YE|?OmaAwe@iwaA)BXVzN+@jW#IJbN?C8y!EcjsFhbWnbN=z z%2N;Ch4EB!wj6tLS!dLODH`ua*5%@ic0aDmqNUAE@$q5`H{3BqKc!$p$HM6j^xW zrDLSg2X)Sx#y?tF>q9v=zZ2zXOyYKFc}6Rg={@lwYKw#hi3c(8L^7iC;P57@KnzSKpM}f`u%in*=sdy4~-XpRkv^J^_}pQbe}R?L_cMGrXSqq5!IqkG|B?Ag7!HI zGTl~mALI!aec#E9;9!itEl{W8v3Vpu5tz5wmrg+%Dxs+QkLzA4OGiz+T(gdC~lC{)%- zc=fF0^^bM=VpA?VYVQnpUX<#GoBa*>Bav(NIh>UYuQmNw_0aDYj>EMVM(G>=MshAD z*)9oq)7*mV5Lf#I9zkwx#TJ)Tmjvx-_$LfilquD9Z&jTyWF+_fBAQ}4{(`NCoF>c8 zZ!Dr|ASAJ=TGi)`B{Zg@!Iko%^XPozixVcfy7Z^~OrV=o+T53cUp1<-lbAV1ojC*J zDE!5;0>tTmar+mTiES;rMLdrz24{DOxo0KCptM41JHfRzJy#KWx7F(%N1it_G1 z<^;ps66jjZ8}GJChW^+F&V+Nm-OP_RpyK+%7vr~>sc`Rdx@J`;@pr``!w<1{@cm&V z^0(nsqbGyGya1Q|;VQ8y#roWWrQc@R&gD&slyy*FYWu2J0GU>V9A@5dTn8H|M$CFuI4*TiRG)l z19D*PtYy-z6{*|2&hjv_*>B|#+;dad28F>jHgFu!cg-inWqL!=Jf48zC zqO0j3=|Bj1Oxj#85s@M8efab2BFItLex_+2lkNL7MXpgzCSMR2Bz28C?8K?s2?>ll|;hTa2pxWv03JVufEgDp0IrP z{;^wZokmGe*WDv-u3+o3>NnJsN@O1c&HC%^f*XigjqFL|l?n7dYdZEkLxodAs#11) zy73*DzftZ5XNz;KBqqFVfIg^e;ERo$7sNhtG0lZiQkrsDwZ6wo)}{-hu`6BQM2T3V zIwNXM;naMmAoJWapssFRVvAq;+v1;(;Q9B#Jp+@Qy&9qC5!U-N4>jjqPNK`WE-AT$ zPPPm-=U)5<0#r>#qNctkB2&%6ew^Pab!sIZ{YB5;A7=`*-pWYgeA5VR{eAgC0mp#& z0yMHr)5a;ZjuDso+3WC&k#Ype{8;JA?rB9ZULu<@8Cm`BAm1V<*8nX&jJ&b#vL71@ zg1>y(GWgjLQQdgR@-hJ<@Iv^C7=yz2D3Jb{x?k#~PCl7&fN$u0TO&m@Xu}CTwEr$3 zDuQF~lpt8OMk(+YyIWy!j(LkuZ~h(J@`w^+@#`(yU#Dv62VDrHMGnYpb4Sh;M+w6v zE}^!Yr>43VGpb(Ps3^*}^3V8$k7x=QVYpquU+_pU&0htl{tFJM19=e{?TArd@tG&^ z`B?bGiiq^A*rg?rFh$;Psrm0D%9FZ|&y}VIJ}XjCZ4s1+(7loZ-@dt$(Og2ipwE&2 z*vp#Cn4~(7+<+ZO%O8xjDvgdF{xsA(T&;Rdz8;fHX+j>NY;Ta8|6twG6U8h)b+jp+ zzE=6dqZj6xK-c8|)px5G#Z*)9NufInHW+VsG?9kvXJxQzqij{{JFWCpgENYspUh+H z?1+0+zukYd(yjXic~_RORugJ2`xRvK=y`rYL-M&Hm5`@4*(N;dx4CmyEmK9KJ;>jq zKEujjg#lwK#BKFvP+uqC=2t^)Zj>Zu*((%;6) z0)+}?xrM;Tyc0-`&3O7Vl4Oacz|vq%!HN(RWB0hKP_lva;h|UWqaX#<7B2c7ilOlYg+m?Pldw6pp-K0SovK0I| z5$oPK*HyKT&FfZ4$X(LqZm?`lq)OMHTD58jr{samH^vP6_OWS#&@(XRtJMBNV7Ydr zH=onvQfupSwa7|+P{qJVuaVQ=V%v7e3;ZiR<_p%FUlwB#tF3;w!OeBV1{@B5jBL@q zesP)h7z#k{lwFq5exG8zoxd+v=&-nG(B8@=f#_d&z z(&8uyRn`RmFy%N#!%S<}R-F%$ZQepeo^T$Nq+6?aeWhS62FdXHi#3tOw>;g3CdS$1 zl+I+7*`#lys*v#bH{98UnSq=&9p<-LHLr}HdirZ_WaBFezCC@g_C(0!|IENq!2$feu;M`==JEu%nI4OZ$ zy@(8ZAwsU_cuMWCC}dZmyNnB@`RIgPIINDR+=7iATI=Y3B3Ot+m3=@m15qg+SHu2TP%>`Q*vAMv~ZIg?pl0tWU#W1^SVfkIxlCV-Hu z+dV0H^voH--v;T8ZqG&F;%-2o!#%=yz^QR1odbKiFUrl?T--pw{ z?sGkYJd|ht2ya&wSyAsI1mGzcv2y>4yen9EK6HoV5i zSxFhsh}QNloa3C?En@Lxipqb54ROBn1U7C}XE(#RM-&mE9*l?@6%DZzxj6ZU)@y08 zVP--YYNpxcf7hc``>f0W$248+mu0^C5&C3CD00#T9xOr&xAH(=eFFN5o%P+{w-osk z`9}EUV&V|kxLo_g6$S`6B~S`v5CqBt|AMkz_$Vv^V|^kOybC8iDp+zu7{AdW!5mYV zn6d%Zji9?svPQ%LY@BzmM81QSoXwEPuX#X1HKNj1&@QVAt`q~QndHGpKpzf}&6LG9 ze8={|O5NAa&U6h{<9xXK%qZ3@QDK}MLa-XRU`;b^kU9SH?et#zFXnVyVII%p03^)+Zt89hOMl!2L<*z>`UnuDwgl z@AMTvj+L#--bcqMsnAmsWnvYo`|qc5|D6>|IHw3oi4Q5-5Ho(gRWDUoBs$pW0zR5N z58>5ZYn^Gv*NGlesnu-NQJHpn_~7k)ep?sk#;%&zJesi{z&HIzf@<`ea#=lRiYy~t z8aXzIi|e}5Wgn&crEQn_9*xhZN{Dqn!+%NwNZd$@7s<7Bq;o3j z{z89@b=vx7)ThFHdmlM0w3^((1%D1x>r6%4?pB;nCYah7`v+|8wn4HHeF`>AKiMez z67S+ZM|_A$RU+E)_rET{Mv`632M$R`z4`Zw{(HZUKsjc>h!n1RX&_gvmo|jJ0#OuH zGK0n@evb}M5_PqUEcl*k%mplxau3WKqGr*UCCh7#50~K41^AY!Kv451W;U4x^iE3O z71GY1)GvjLfVsB}N!}^ynr3P@v)m~=S4do9j75ZYkk+?4F&u0~F6t1Z? zEnYu%O%ZH;Ie+y!3n+WaeLs?TaBF?cU{XIC4x{AJjH6(k-&$ z=W1w~b+O(j1w(P6;8lp#sW(%}IXAJA4NEi1$u^t%p66zIwDu0d=l{flEf}F^f97p_ z=>|?KjyUHBB-8A*A0d-UFsg&?Td_sAEswS%eBM=F5yah5- zvcf**ZyU;>fAJB`QNc@OH0cO*Wo4g777>3&$n^8H6*0!C8us%M3skd~Z z>57|<<>)TseNUC>{r-_C_QcKa>9pK6HusEmGzR_k_7dCt=d*RX^+6x%%@IM7=iNF* z=5cna4-SWGgGL!5Zh0=RYYi>C>eAfXY=Wv+&VoK`S*C~5n6lnwn zCK15rQ>?hy0mTE${WIbf| z2-=q|95h^(YEu}5_^_A6z4Sq(KWpE>E$H{OctjcDC-cplnejl1(>E!DIb?u>jJr&_ zOoC*2)GCA2E51a9M$NX3^UX-(yxb-ypy^ju&+WSTvv2H3b|f7o`|qAS%3?$g-vKzW zzmXXT2BWbOJ~*8jLmgZx@kS7R&)$M@4WY7Ke|CYn@~X1dUeAIzcz#B%baJtzz3gHv za7@qayA^lCi(FaKeIRpQ>}qoQl<}e}DXOf_0+%X->B)%;!iZPtU5=5B1<1k%Bk!4B z+}-C|px|d=`PU;L;d}eo^&U>*Wl!4Pr!7SJ8gAqL9FBOdWn|N605mUK&zAfp(Rjsg z#gDdh5Y?PV(nvKaO?7AGHiJpCb*FsPvo@QmyTFd8ajB|-h@bYo=H-}bM0EN=cY)-7 zLZ(58?|L!5TLeg!^eb3|N$!|XAI9&V6vE{EHL_L2`SowY^SIv~SE7FRC3fGBK0_o| z>lP=jLZ26i?T_!ZK1nCY9OQl@gm!a)jofyNlQ0qbA1?qJthOalv5CNr{&GX6W1Mb? zOm7Am#i~i&7vnTVHUpNZKF-!fEG3oJ0#NA0=~r3GTg|OZG(@0(T$uEm>YCO+T+0I3 z@(?9>+$|-PP<`H?gLj9r6X6*`50&PXG$()QU?Z^yH+*rnA}QWG1Qn6*IaT34)L03Ys?aIcbGgEhwYlGhiw3CrhK>Qz?ASrTIOO-o6 zAZmCQk1Fqr6p9!E41A&|PC@gMzd{HQ0%s5oxzjcyq(BmK_)3e@nSA;`t&rWz8s|sS z|MZr?{|@1A1S`xYJga>qx2Y1`jNPmo%*OZE6f0WRxoP_PF zz4&L-&|!Qy4^NrK=NF~7z=wJ~8T!RkP7z`9qZgfXsz3B}l9h>*qatd$X4o2Jc8W^O zaw!&`V4Vi#ny1(tNT^lqp0v-(ITd2Iy}8=y#Qu7e`a$gIj)IZ(?4{*K?slpE92;M3 zwi%h2s8P5G&rF%*O`~ScMa55%fz(nIRgwp5nXd9}8ifh0FFS@TYr`rd^^PfXbtGs{ zAMGLs3+TjB=X*4&c3HE;R4%nfP4*_@UNrIZqH{@DVt82MX*#}6ijQ;fl=#ub@zjdM zby4epHGCA>JdcBE);GbB;U-!IlA0jqBJKGlqSKRHJTq!hv`IOjLi(&Bub9#atF9 zEq^-Ajt|&6x)k;N$4VK=X)~`RuD_b6*Rqz1%u&J)bij#dnT6G_F{xF|o2K6Q%h=eq z7IpFXFTR`~FP00yT@v6)ZQ<*^lQW=`cg~HUP!ov6s6iR7qDtrIw0|4DDM_fIBb8CX z;6ckO$~-g(XX`c4`3FrC+O;|hUv|<; zO-?y%b$PL3w$5YKkpk8~y+eZ%*$`X!UG=qu+;yRYWqaq!#iZ^Q3)y9`{x0|v)!By> zYVFz}1kk0w0t(@!(iXxBh%{cDRH%iwgm-`u>E%Fc_X@Cp6%jO0_kC~*asUeh+VkJj-B?Y|XmDWqq=(wHkwuA=N-L87=o~$pqG* zaV9Nh5pE~#HLYl|x0Js?T2SgHT}mUxB1#9{c(@L%%xAzgY1&qyLpSP{xIQGgu0ZZl2CUQxL1 zCAq}v@n9n@mcDKt6Nz2S;9~98EHJ01hrfuK$}YU*FFZ~4MVXDcdQ&Ms-t$H)xpb#o z{;R$kgf)i#x!p+)y2>jyoGe^ct6J$krYAz5*axKlo3!L5EjLDufr3dUrf958P?RSH zk?jMa{>l}mDi0TWd9{D#LAxX4B4wl6#BH|LPj&DXFHUYr2hPZww2e#0zar-){+rs~ zDRU%eV4Ih+O++Z>PcT2?5S@HKdcyIjh)83^{)U~Mx``T3>zINBV(&TCSX$YP0|{j3 zag7%=4Fl9p*5@q-7k=)Y4$hKCYPa37>kG0LYPjP>FBEBnpZ!`8p=Ou)9WU-JsWnz9 z0Ofo^MVpWWT6w_o`uyaNgo|0)Kq+ntwO#%y%@51aTmzW-g=#&B$)#K;J!kdZPRf7( z!T)x{4ZSha22{U-_On#iMfK~xJ4_g`*>vvT(7VOnSmYtohSkl0&17MQ9JvFfgSOF9 zuS6sjp%+RA7D!0g^PzLgj>+I~8p1aLqZIRG3REc}cWff)^x16V;1glWP9wkNm!R}gD0%Gw z7q^XlEm1?9N7DRIZp4WUo~lCG9S<*;oR)J&?8Ci(ClL`vi-h+u3?;t7107!y&;HBI z0QrtUGPb`@5(JA1RpXTDU-_FkVGKlSUzID23$j>)Dy_c{tEiL^D0h9|p}%%_kCbRLO`iPp{M-$PkBSEcGg;|fzQrd<|7IN~l zf1xMk$?U59w$MxxGZJS?WT$Gk&TPh-RO%U2fC!b*6JhLo(;($aL3wC8h>0J>>jR0F zLO(i&=vB`mANl1Qduq9VagBd#zyL(z^lF*fg+?| z?l`6>PS2n0N(|g`mm-;NTsmUwXbY}J9%hHBd zoY0thGP*a$Sk#fASm}=-1Bd51^2@)-&gI4=gX=*JJbi-;sv;Hq3Xf7HVK3G+OZ9%f zH3(4ANjtM>N>}~tWQB5gXXk5aeA-}WWg_&Tt%R*yaJF?v@#J_%x*6Gs;4S1xGEsza zecA|Ci9r?Q9YWJQ#N%kWa}D9+RZ{ynVCW5r!VGBsRiVuX`Kn=g+TdaSxRh}(9bA8P z+ZQRql-TtAENKahe8vI}dcUoKQiFnwxy_27M14_F#mXBz(~FhgUHhre`0Bme^qb^m zFjO3+Z5ZrZR(c|!CWMe;h(6Nmy#zBR5u$PHiKNbfiN;StVJ^^5+kD&Yn#N}QE{~xk zXw|T02ywo4aBI2%qJ(-f@I-^ejei5_(OG%;Qp!Er;YuJ~$#`njxdI{o*C|)e2+#h+ z`>|`2LK<26XqY2vBH%HL7Scg$TJ0vw*M0^0SHlm_F3x8! z8Tw^a9MiUhak{UV#z1>8!fXr$oXctu#*-yHQ~Bgktmtb_)@{}KaV3>l60YZVsoXh- z)3$SR6d}+H?3}0NSnbxVrI*)LilXAykZ>Y{aC&Chm(em-jR^Fw+7YCp9^P|`|6c?R z>`JMe+IZ;bkph>3(MOG3p~)n-51v9fYeC$kAPV=xg4uA1iY$oVw?8^&%0T!4)mrAp z)knu^jaE~|kuipEkyi1hR{hz>wd41HyfNw^ry%o;2$m@iSu)W#w_93T;+0w&nJW*t z0qJfvEelZb{RA~cX%9OVRl;sq3pug|S}hG;$*7wav_Q;^8cXuREPu5!0*Dx+2d4&K zoCj856F*LeIa6|Lw&s~30YhUcYAXl4EY9Puwm!rnJ&@#xT;jL6oR+XjIxMz z9D>!4ZWW>M;T5t*Pih(O3S@_OdELuJesR)ji3B}GDDW;~JbcQy4gZYGQSIVFi{SQF zhzGa%Vr+v)6j3d#bcD9?jLr*{h|36xi^)!3PKA=ve3~Ofa=m z@OCHb9+(q?=hx|zb=mO}K1n)*8>&(Q;_qgMrd!;p-ICJEnfwsisl9E67NGm|TP(7` zuzfM@^sx<{%t-uVR?5CZ+ac{tr30^ZVWeQ?lUba2uZ(5WoW+LBD*lWTa3Pi67oV1; z(l@hCW7V{vRv|FsR_ykcFTArp_|$X_gheC`Fj}ZCIx?XdBD}4u&4)%B_%|pvHJ2^O zUHnLY=-k^B{gp){Up5AUpbb>ltTKeQ=5gd&1LBmMo42VO9u)cU{{R9({k~f?|NdD2 zg**AAFDxunmZb40PvB#fRpCR27+E&{QCZTD`t>jA_$$BK*tk(+Dx7C?kfYT`wRN%c zzc{V%i@d_uW=$Js~z^-*?FeTp6i3=NXp=al9QX>g-}Uj^b{5_1QVsyx>81_1w!haRUJ`YA%@ba z7%yHe&v2hQq>1&X01}i1F#aO^1L8#dbA%8|MEVoYNLUE)&=vVlfJ5ZJ%Rc{v1w8o0 zihA0N*|6cT7!A`I%Y992fZOLapBt&u!7smCTIdi*f)Ezc>REZhOz5NJIl_<1ulrn} zu=ImM$QN>|1kL|TQ@`faJQlg*$A4>6rcUv%UJ_ucs%vb`)~#%vb=IcL#9^%yQE2t@ zs$YGnYkmFxt8E1Zg;rc#ZP!i~p94I+XyjIExRik`rZ($({C;mxD ztEo~Rwq+5og|5E3s$6eT(l!Yg#Vp`+a`RM^)l!cXB9)a@E*xq+wI0gP&-1>jthE1L z7VrF5BK|S{w5^(Y=OVE1=+{s6->V>{+MQa8RMYM+zxu{1Dk^|!9&M`ej<;>o#+o*1 z#Jijt3)ZSVl(4k;r^eOg4)q5$QfRT_m10@(-v>x-1US>DPqVMS8E*v$F&bBmlg7P6 zhYrxRXfIKmrMN@34?q0Gs;a82efy30(u+wvU#2N6?KRJ-EYujc$ zYuTa|4bv!dEdc+e8UM7Onj60Q=4*TV?f2~0yYFhvn>BR{3H3kVnmJ$k{)cJy`Inzt ztCp>8gSKseCG@cj^{cIQSD=4cgD6~9AYXp@wau70%auf;;Y#@O`s=P^Ypu1WSyQuop&-FxtLyY>5jK6S6 zN|}ihC)@KczwGPJ9e3Ey+N{66Rk2MswZv0d8moZCx(hE8OBo(pHgD?mo0)nYc>~^t zj7ncuf}Wg~cyN?e1zet=yZ20OSD4quGi7@H1bXp`^~@nv%0F}u2^7xd(Z9g@=}2}D zl8$(9LdO#Dq_-qN61VuVMy>61*Evf$aQRs?xHI0skJO2(h2 z(9F6=Y0^cT9+RSM9xut51wn`?f__f@S}e9`+1z^cJm3EF)YC2m=CP=(tf;hu4m!}z zI-`?+B~(*<5vg`8$NCr7e=S<2aCrNj_w3mhUc}<1z>YrRaBGNwKCcD*1vY7mMZdh{_Cx8G4;exkUjbtO#e zR;_H~4L3lT-WWy3Jg-Av%_RtGyowMWr%jt~@4owi(g_1BUqj|ID|HC}!Fb$j!zckKrj@=^+|x#n8dw%z)+YR^3X0`&nkx}@daJcI-xhLkKSPs1a{Ooxp(vORa-ofyTFK1B{a6ZB*F7Z&+N zD9^^*Ila%f?`fBW&U{w{6kldUHrSw@?X%aO0XKoITWdKSvE%`de)xWhjeG1r;MMoG z1<;2dePXZt_f?xRWx6$K*4Vb$a!cE8+ik2Cg^5^p4%R~{{H63OMqxR1 z`gD8txtHv-&p(G2i><>(8`vJd-pv)DHMAoaTv!?WZ&C11hR^Gl(m=yhjAg~OH{5Ku z-hR9N`KUiy$CHl7LITAs1q6Pb4Cc=mf5D`2qf+bAt*5>F-upK8vP*2E4L7jzxhPKa zQTjqp^ReV>hA@B6efQh&;UjJLJ$APd(1>`0_(hP=eQ+F0)UKc6KRiJjH` z9Q*HUui?j|)Cw54Ind+DC!Js?9(SxwojToVHtoL@WNR=tTr_Ho-E!+~?)xJ*zYyH1 zwC%Rr-um@9*V`(Eium#3>d%IMI{z$={-t1R27W#~?lJ4#r=PlQXLUK`kMMXe-=CD9 zNr-@XX*oP6>`FbL8WDK7JCUVjst~SNe;siE4TnjDf5bmgkrP*js6mb7InrMS6p{Z# z{=3Zck5<_dJoWW?DqppNESD0Vv*d&?>lVILrSq$YK&tb|@AIt4mN`yesKn1T7iQ)^ zC04LjV=ga-_)NBIm7#d}%b)*foe|RK%$Wm> za@^GWz4zaDCC`i*)9v`b|J^#Cd;$uI3U6K5s`KOhSN*E10L94c*|Y4VPMr~~zUNCa zTX6U?F5amxm@@Tl(4fS&-g;|0v-4>zyj#1ah2jeM`usAmyL{|b_Wcjv+uXTx{i|jJ zobeU$C49HtcCph=Ie~?9b5~+$k9j@<=XBaUbofR7wYIFRjCSKRAHlsr2^J>B4XoV; zZSADvj`FPa4`T~O&T}0nKNeC7oUG^8*J3pZoKXat1PcV zh*8}Vn5bN|nK$45dhAK|?Zj{G;)_Pu9=q=BFeyNxs5R>9(0?fw-2wyN^!b-xI^MQz zvo17L3?2u(2mtja|LFKHj{k@r8nTU`|WzmS=1wkY5PZ56r3%K6l=EXB#u> zBHyOr{EtBv{>=f$pM3gRyXWqI+7DBwIlljTx83Z5!2_`nDJPBiSm1w7lUV%xFlCwz z9)6*{^Y**YRT->Tz8gwytu@!Qb9#9(fTeGa9@ps&$zf$Oqx$0UJ0JqzbhaYKO&pg$~Td)Wpj((c|{TToK z=9>w2@@buc^R{;3(1DBraFOiBpW2|UHmzH>u-ops+XfCCX1f93D=)p6v7VFgoS#?! zqUc{rUn#hxY#TLtjJ<-L9rO%4f1T%fM79N7O$^cb-5+j%1GXtUrkM_aP&^RI}h$v0w z%2rnwfu1)EvRIFh&=Y&PM26~!NgiVT0ZI}64u^<;PEdh7oR}Oek^aOk;2Dwsh`|Py z8u{r{1S%QF8dQ6D=rz`Wdm4&|I6q+tV z!(v!M28()xS5@MRbLCfBCQZ%7T2SdLs^YzXFq!&7R?3aE{=^U@ETEN?a#X*}q#L6k z(SGVug!_vwyWFn4@*4ZY9}Y#R=z=L#Y;zcM#dUV`%{N;sloJEax12-J$v`H7qB?lWRs2U z;t|8F5n)vbbQ0biRtfxntbV~x+c0L%oNcF`aTdba54P1-TiLqnu7mYR4W{Pv?dz|< zvd=#I9HFbs7uQ1v_vedx0ZNiWd+VKd?A-JFAY9G${pee7v$Zv8(!@Uc=p)Zpf$xjm zFk2ryun$T)`s;8v>o;(yJ@xE!2yi9#D;DfB0iVN{*l)1p`0#@dz^!R^>@k1Es^@ec z|5yI|x;_8mOTL!Ue)}1-X4!vVeGOryiuP_vd)Gx-f{y^c>Tj}12m9Rt`vLCo)mPhs zl4ZjZ!C!5Z@af7PtXXExo^74GcDD~c_|W?F>S+h;ySL3kaFs<*_V_zq0`pXOR{i?# zJ}Ve_ape^MeSDJf7fvL|@90ye~Yrv0yCoa6XnB_q9 z{-ym->0h}-!@2MYDc?>xwKMPYZRGI5wh2PMgj;>x*Z2!U$}f0j!(U-({odHjkDv-~ zVi0BFA^y;%YY+P`ioxA}y@zesu3b@!#5xJMC!02KV}0mN?UOJWj)gp z1<0jiuC(i}yV3hUbdVGZ1<+pT|2)dBsh(#oS~RtP+;guD8a&i?-+d3e?BWY;HcB!1 zpGoP@EwmB@bNuC5eG$$i|DC4k7I;h$c>{gO>cWvIq!nPu*AS%_W+T?DNhylE?)KnA z57=2}cCk}VI1Z~IDN3Ntq<--q)m4I`TYk8jHg0G)-|-I{GUP(rVaH$DkOAjA|976E zJjoT>7x<4`@Tj?5N+I~t!3_WLK>d(L^slm{#QN@sDLzL{fTy2(ZV&q{JXyY8^0CMX z7iI8oLHMigpeT`l6Y&DklJW$8rhAls0=xt4lfiNL3$hxa5GFLMxBg{tL{;@JB~60_ zd`BL_vcv0TSVWvoy%GLsKt-Lp2Ygb=p(GCroLIEsp&^G5UNw+{@Jxew_OEDZ9;HxG zx|2T15*|ElC@t0>;UDoY(qH61;*^p9YE}>&R_6R8Aw!E8ZZ?&zH@=_(AvN!5NYYVJ z+gLC8yS@g=bln$F2nqQtI2B)od3-fOj`mHb6Q8DW37}G3s9)8UGWjN&vV4xDf@1PZ zaFJ6@U0!ukzY=M0d454IX4p6u`-VDs5Tv-__FiQe_2zSqtFS{D$T)GvA#JUyLm z^q~fL72z*n8m83U&pppR{`h12?Sa4bue%avuDq7Maj35CR)GV=i2WM`Yjd|IPXL-kk5tt{`G*}dfS~m&qcU9-}c_? zH!dWe)vXt%>F@gf^X}cww9Pl|KpP5d7RrSiZn@nadgvjx3{+c>u4maHlv#_?>q{(l zPVRK7k4eXlC)po<|2vctrC735*(@w(-sTJX%dfm<$NcqAwnocV?vp_dNDILUouZ&G z;GZEZo_$U)o5EuJyz_ce_CanHQ^45EibTE@(*6evs3n8H@J~XH;}_eQ3Q$zc#!9Al z-~Md>`OJE=;6LbqeKD`+t1)oXrSTWCs~$mKTPb9oFNK6cB*@Aih431w0DMy_=!iJM zKcy$pmKKci*&+pDRc#eC67WAL>Vl9@5nKr7z(Upvg?ypS%MrccONM&na8{>8IO2+E zcohESfymmlP)VBR&6?OF4?k{w2M)4L*&=oQRb$;6NA;)uS6kJ_V~#%&i;_t;Vi<~& zU3R37Dimcnuf&2vru`CP9JkdM!Hzn)IsFTNWE~~pO--qS{-k`%qkY-^pC^Ti`sEn~ z6}EJ-6)T^9Nl=qAL6%NARrumX^~e~3xneQ(Dee}XGA4nOPwS(}H_2#z-N&|? zHP&3i`kmj)wnYgnWv&E$;j-wjWs4Se2NorR1`T8D&2Dzt$l*T6$cjhwtSw+EBvO3O z#=q2WQvpgJS@x++IfNArg{@9%KP5O&=?(wo;9Gds6ovFTz5241>rp$ab7$*x%JEoi zfRD6KWTgHpqxe!S>6NER<3@J(egCpcMvb;#{b~mk*nPoK&5a?Qs>z1Gwt5P+)IQBE zQbdbSt05UnrqD!h8#Ou(Yn$cFnRA>+%cssw*IgA(tkI-rQZQ2=k1p@3nG4K+!mk$1 zQIvSU`BA|nc3A2;1o+*2&%T_SM%D>}dP}oOyajd*{6m{7dv4lq~)G^tOY3cYw{9K7;W_*kjv+ z#`M%KUHPK@p8aOey{zx~=OCaJ+SAWGYlHcssqwh=rfVs`*tZDC?}Zc{+Ug@;E9$>n z3jnvUkPZqY`7-$$N2|U14nR@zgPqg;Y}y-?7h0fekmP@3N&3Gy`0JQap@?srK=3`c zcR%~&lTYltUgz3@`|X2LW;QqtLV$njXR07j-y5-2NCMUP@e_%gXRYxO&>Dr06izjX zGs9r@At^lcP3_S3rs}F18$aP&_XEwfmx)}j4I4j(CUq$rh@Kp{#EwQWK8)0hCOBrfz{@C#TZ zL9$WfQak<3?)CzfP5t|vXMZ~KaOYjZgDSSAR7gQZddCX^o3bRNN@@7d`~PkGgQJ_Y zZ*SM#c(Yx4=@>iUfc;UF^m2aWU)9MXC0Da1P3^u19&|~z) z=gk6?Zx!-ErqOkvOJGRHNfRKa=kP9&IPjA=h3ns<_@C%c#XEl&@x%WcVGS{D#!UO> zoAEBpZ?xftcH?cg*|pc+Xh;3=2@f zU#KgAzuKq2FTeU49*xyt<0kl@T89SJI^73L549*6D;@;D=HT$-Pd;rudYxyTI-bn7 zo4>=O@NMKwOf@(p7CHVx2^rG0K$+_}05|d+f~Jk5EsmwgiMw25VbbT$l>V5P(z!~rm{_v}@p4SyZ^A_StvBB`2{aOz*G9k5hw^2JA!Pie&f~ZfjqN0Y~1ulHaQbvnVFG20|cB&el zf24jjT^2ZVfKNxZ-e`+O&u(4qpab{w6FKIq%w$E@>-@fKA9%y|+izdKO82+>|MifK zxOfx`_GWg`h#@8+*tc(J!A-$(2zduS8ODsc$~M|y0~<4Xq<>X^>)m(l%(HvauRQB@ zP7gZ(0kTTIRRDyXSIQ}-3nTQy*K>kH0$?fKab+oNi~?fd^pLjX>(B zxv1CbwhAPlE4ft062d=VFv19V@)5Finov*b`uL-d?fl+7QOckwL9ygpI=o%_tfepB z^IQ;l_^~JL-w!kBQU->^sy)G2k1#YQ>2`d|Cl}Y*xmkk z_#v78Yy6!rCjFNcm;4qy`0%6F2W8WyY`wYR>dO&=!`2%Y%mAHWPWSDYzn|pFh%*q% zKLQ4C;?F`B9<5rmw4HbQmHqwLKl|5Zef1apIn;r_gs9v9akoA7^m8^H0j{7ppRFQm z+0MJ}WPe7^9FY}`9z}1GN=1Jj<+1uot}PD@tn*pj?fDm8VC%v_TYsH(?G^Zu6n$G@l_ST+ zvbGVq2!@Ut6e1o|iV|#&lv!B5+;saLHWr^dQk=*-O9~6Qh7r80XlqLZ;G1v%hh22Z zDBEYRy?xt@gz`^5{S==>y%9J$V1o4f{AO<~oVt0P!arGZN!ccUC@L-Bq}tukE*yD@ z-T&`@;b&+Y8-Bq6Xs^)6aE(?it?!@<>`vPBx4-|*`kdPnA%CJfs}=2?hb77@FTG-? zAnYCs9=Z=C!ljT7a)}zgeEDBWf8u|-9F!=!Zol&`dledy)tjQ)Z?vH+Sl@p8Z9D9+ zL$RvBM+tCJE}9rJ`XC6ufW{Ki|9tTnEZE*>``|}d_OyV%ZM)qz)&RaE{v*70*lO!d z%jR|`I6iO)TW|0=v(MhYVN2V;aauaX`PBOBuV;Tb>QC;RTFOk{N(XXCAt+0$$$T`B z<=y-5z32WrnlS&h!M~2?uI;g4DMLXf9<<>6Bi}nxsGTiE$s^~9c)PdzJV5e48w0Nw0cYc+sgh&da^Pv><=+>NH3h<%|dOLxxNqRqWYBTFHc^Fy+%tzX`%BC1hu9 zJjJ)(V3FWL1w^I=ix#60pVl_423>&C){9=}_qW%yM<1cQC5v+%H!5oYZ9_P@;|X@c zF~_*?jb?lemjKtdZ$I|eZ(`S8J%%(jd?gP1tRH*gDeFIQASbG9VI%m`D#1YY2)9M% zvP9^MW4#w&dYMIGLz{(lNplu=+TTBs?E~#MYR@)<%bisOfrzhzv{_$_wfFn-tFN{j zZ@2|T$syLAec$0&*o1Oaf%=tP;Su4`=i>VP_x1=Lx;%=Iba6)^iqQ-{U(Oa1n0fJO zT8~FrqD4Cv9Xg^?U+1(ymTCUXS+nivqyJ=GPU~nDE;vz+##FmOfa!wu6Y5uhQeFM? zP~7CPO=t2>kW`*FeOM!G4vwGVGhJ( zUjn|A7E%aKnKISC#%{O6c6I@RNu!3PY!8toPY%itzVbG0Y!5v27~3B%b3Yq8F+#%a zbWReQF=Lu-fUk`ag9jl%t>;!gQmV))>v_&N~9>!)|)F(GO$em{0n$Wr@+lzaP!Z{o#bXWaAFbXWZ&T4V}-ngB(OSw&`ggrGWA+ z1)^QyoQN{v52cLT>0P?nn{U2}rO$ed)ih2Snd%lWqLKag-Pbywb`r1&)<}Z7U@2;I z%QQMFf$@f0ZnH6CuSB`GzYXk*qNEy!tjg#Xql+#XUoTu3(BS+YdzuGQX&@kqq zU&5*-J%gRzx~Ha5hP=gKRu<#d}n9pB}MSXLPFV^kvL1$^RQ zw(oW8avHduD&M5YQ7v2=!{l0!|1FOGWtCTgWs|Jwq-c}FX<6Te{Pdc#6)wfWohbE& z4Zp}*q0E~-XC|=8u|eG&ajE$@=#0W!rDN6C+v*%Hn*;i<#qI3d|22WS!Nw11@jTm?+5IIk3X>lc3W*@dvgS8A!A#ELM>z7P2064L&8x|l=Q`#Vv6L)Scy%)A=uh-tsdOoef8liC5CDwSmjc>06TEaq$wzFx!5@n7$+zC*gZaR5K9(eM zlsFA%qNgE9f5-Nv-FM&BPCeyh1g0kT=;Ke?O*h@*iiN)&^H;V@oa$*cj6`@skHkAK{2n{T<9b?J19Z#fbEN!h5ex#Ak&i3P?X2OS9A zv_z3G)2`vT)W;ut+>fc1UjX+ZLSKR~5yrn7fvI_mX7XDGA z^_AA60)hkOLn!}B*itgokC2wKN3}@-c;`L;w9%u-+J+l$;75vf z*r>ft$M?>QFT7;0z44~Ws%1S)|CL9r6mAHPh42QQcq6B~5}Yp?c>#F2xm#aI*`~f| zTZZCzglRE$z(MeT>z()P7Phm<8tUW|kF%|}*aAN;@?|7zxL{c#O9eSg?bPK=d+*)% zU18a#?fM* z&+8J*g@4uHQVT2vZn+bS61K|xX3stC^2;u=f8tl7|9~O3#a5fK{b?7w3cAzwz;jTP z)X-M(A6F_;t9Xp8kN+LUp=XxI?9tP>8v^f>9O0le}hwwVk;X;eH0In^G+6JXdRazGz6#oFOAjMr1YQiXjy&u5#->H@IaIyA2<3iAl@LQwJb_*zvC_}b}n_Rp^Y|f zk7dX4eq8&1P!ivD_dOh~Jqe4CE_TA-{)!@FIu+&m7+lR5UUU6TY+qa3Pp#PrMd}A1 zf6TV5vG`#6%=X%AFB{aix7Q{7@p_Z_M~cd3@RqI|ZU5LKk64#8&$Lb_qbNy^2VPYF zH6J%d$#mC0u_)n$qaA;>JzH=3xv)6U|9s>54VMSc9)&Al;(IUy*8qwi#*b zjmFE@9A1m^35(KkRDryaKB%z5dl+M$GT-!NAf7BP-q?7Zbc-vU=~@ zdZWsusL&+kgG7kxKnYhzTuP&qga>5Esx5g8w@`-CWis%n0rz?c6j6jY!W10qkMNK9 z7wIqZAK0NY#K?b{B38xxqe>;{Yh^TL#%yLL3HKV4bao?NkBVU)lX>y+C!b~?eE1PU z&Z*Yr)RXz@J=OPb*J3F$3x{t_`PwVNM_;_;JK^DRk6FL|1O3E?tFIXCUp_Ul${bo2 z3Ip)dAw`J<_L2rg=t%?#=2aG;Qg{p*I>P?*)PGoL?#;;%7r1qT)=W}_=ong^Lg7=a zw_8&zi(Zxw@=}IjGDNt!X#*Di;G6omoduWnN)@rJzb=Jo3yCZwI-b@gd?9C_d;?&iw&*MB_uqeqgTWe8sqkw>y1 zKf$+o+{72&E3UfAwm={q#Hlt-*}txZaiGyKUe7%Lf}P)|zcuG;>CM+)?Y?CM(;~Ks zd^>R>C)vDgue|yyg2Cq~I?6bWqs+Gj?6mWa*0u8)Y|mQ5$4l_n;#!Mv3DjTVqvYH^ z_$Zn3z4h$To$WjOpeP7d9vYG*?Z054@eixF1ndq(MuYwfHaf+k*Li&rhClIBZ`>b> ztXEK22-&serkF3!&ph)yiiZJMT&!VZM_+7fVWO@t$}RcoJZRWR{2koxrui2R9qiT@ z5VN5(yhv)BRFwryLxdhDg7Z|MTAETW`D5nhVqWQC%U)}=4=a;REbsxRt%@#gDv zOO%W+y!@)2dTJMI*}A1&HTE*!o+V{h4*f{SU&6R7O5~%Yk1I+xwd=0IM+v?z1m_eY z5;P>R9CQ3h_6_hp;l$&tD@RmMoi>$iH?8bn5068Ebe^q)g@aB7k#eH~Ukpv?Q{TZu z?2$(v#q54hy8wrg8gs!?3WRP5ckiX>8xY@IuGzldaOl#@uXJUotcn_<5DIv$ ztxT2p9{R2I!Wu`)<$xbjj*DtkpX${W>i1uMRDT-7V}^E>OG?p{|0t)&i$SX_A~c@L zFJ2;xja%=yo0EJ-TWgL*9yxR%zCt#Ie((tcz8(i$#(z8B4(F7zvt&^shrbk)DvdJb zQ(%h!4x_YjUjMyP@$~=zKmbWZK~$pYKk1T+@HS?#Y{x&*QOlOiaOR4S60CCCwQXmk zE*@d+*lr_5YExi->)I#+yDg0e&BSi^Y(9~hINwgT{Z&6&p z0KwStQ4fTr7%aN-G)Xv-VIY$6V6BmmUdZS13;lR0A&1lDLJ1ZQ(AFDWUaT^yN!-%K zfWLSO!c_D^b*KvoWAP@c3@^pgCxv+hDX$33Cm?!??y-L08{r=?C*ohEKOm4{8e%A1 zk^e;gyQ1eG2>`5Ge3nj6y$KVu=0w3?h!}(deGU9}@+6KR?d5{uFa)2?H{BE=kmD0s zENjs%;i;MhoEAdbL){b~4v##>)*JT5x9_mAT|MS9UnognFDYtZPdxRs^&dPKp8#9n zU~i!N2k@_c)T#0$v<@0}p*_us4IMkOul_GbxzMAp@Y?4ulY7+_rV=g4C7@_=ju69k z2U$qmkoqX;%GMj&<`?B7Hj)$pi2nOJB*ZAY_c^=&X;_*>Umtb6=zR}9Y>#3wz+zF0SRtQ!S*1PxUZJ(o1$YEhCL8TvN`+K1!+Ud;ggR(W$Vp3e04`D1H8i48{s-uF7)*l z{7MLae$ANlKfjlwfRtSFjq@-{1u5`0#kbDz!2_%TU-E@p#YH7{4@YnzWB6m6vAFFI)6JXMqgA0^jwJg^i6Zc!q<6Jn4@q<+kCC%ce-*~K_>MS(FLe<*QU9gE+C>+J}( z-^fRY6ff#;F-n9t-gt*2c)Pi!hn#5c^s60g!*=aBd8C!K-(W+=p3`=u_|bSJm#jpJ zPzrqU^*1O=&T*U`jxuWVO~N)CDFu8~>681HkW@&=M0Zlyz3}oY?lVM6m>%7_*-krd z>w>H@iC5?n{}KM_Xl_}XsQ=oEB;QNT*k&dE>yZDQ0=Av%# zm=buvO}EIR1{MK-I{Z)+qHN=U zp9C2u&_BN@kn*9`7wkVzKjYi0w3TW6w-bB|(*=VE*w$NafuASgJn-Nz8~$YX`e}m0 zUTsOkc=Ag@!ejYHxrz4w_q8{8M$t%O!KkJR2cTGIbUg6{TWf8OJg%w%uWGSEYz3^Y zu~C;@X8Z2FuU&f4Fq_WyFVUWyqz)W1-1+H|M;vBdWl@5nMD;s2RHez~O-gWDZA<&_ zi9LXXp59Z2Tv?AkmRbB?ng!97aQ)7^?zI6}!c z_Xo3;ljG)TgN}!m!{(mn^|h&3#2taZo;D~OHE!C%GM%kXcl_gSl&o_(Zg_+{@1Dc6 ze0` zQVIq?LGY0;zO)lNp5`aU-E!l#u3&)#oYzU|a?&ZM(}(f+yBXv@NM^DPS3ai%Vy`#7 zhw4Mn62f0=4AHE7lsFY7^cv;AfRF)q6Xf)TOftp%$A!7@20;?|FJkLWGD1E$ zdd|X{Bucjh#xpA>LDeiLLt!;BK3mB56uw0O+!e=10Xg^NxC2o5xd<9FZ|1Y?=}D~2lM}w{mI)r^LioS+ z&VRV0)=fHeuptBc`DrWiN2B`ZBk)O3kYcOfz`=YOZfUpNaE*t#pi3K4*hp}b5UMZD z5}t;S9AytZ{D@oMTzmBu?vtUa3a9!A5kaX`#EB@fDCy^0Z_e%B4WV;i6h?{if`M4l z{s;aa#!jQ@SMngCM%xUeC^@%JADp>doeSp}#IU9$0Hg{R`!RQ;emYxlWus(MDW2%=XxMCloU?-C3&I(6e_x zS9D#71;-YYqf>aA@umBuj%VO_^HrSRo{QthBlyxiU8VZTGo!{_X=B+&CgJhIVRFPe zoh?qZkH*Qr$KChbXZQX40pGG9^3X!1q_o6-wf**NTk6ENhSn}b3(u5Eic;atSKmx< zMTu~F#08-}@+~2yWaxSLts7_2E$~1{mtP2tx18Kc!5jLos!2cw!9QVrJkwUvhAn~k z@B!X$Fb-#S?cqLlZn*Xe_syWO^GiighRPI8+=!xJ4E^_0Z~AcR4T_G`nQRpTa0`5t zNa=R@*s&bhyPhj;=2l8*9*Ol9V|CN5D8a_qAFz6p-<3*eQv65s?}|D2#naEeV1tGX zL!mbtUUo3)&c>RjJn)FnDuLVk@BQONN(?EEs!;@W!7}I7S6;E-v!z8!zZo;}M+2X5 z9z#CQ8wSTa_fz~&Iix5Nzt(Z*|9SE$7s7)AO1vLmJ#4kOh+~(x-Fi#kG9{&e{9;^& zwTmp*w7qWhs1Z){;-9iKx`b^Y_o9e6;)o*?MG1;+8s~~9^-q9TW5bXCrsJ>RAESTe z(&gMir@(|K@3jV0bAv2iq|^|s_t;|(8x21$g9gNZr1-m*ZJgI$cRg@Cm@TH9q6W=3 zKw0+Q`yc)vd*=b5RdMzG!}dlOQB=eN*bo)$8hbYyOJX-hY*Y~`qM}$q1r-nr7OaUe zMq_(TJ~fFNjU5#bE7%1@+AiCH@Ap4*@3Rj?-`6Cvi#QkdxpU`EJ#(g5{Y>P(!VC##~f#?)(bfCwg${!Ot% zKoV@So_$a>E`p}NND?Vc*5!cAJSR_SWO~oMPXo&_=E@=j_6m8mkb1F%Jfy9NwS*$<8KjL4Ue{ubZ>u*!dmu%L)5vi9rr5?TB2`H1)-h3wTHBsLp1xZ=9aq$hS-(pzl^yKIR{1w2S@F~t{+Zb z@2meN{nh49UOyRX|JTK1LuKsjv`t^?)XzEd=D9o9To?nY-?KnK$l#`I0JOl^UTXNx zr35&%zW4qIHhlO+?q*fYpIvs|(R%ki+oeEy^*V*Gu)p@Nm??AR+v?9c9eD!yYV5jeuI7tsH*kh>k(v>sAV`#af~3kNd_nyhcgE*p7!xTEj1M|d{4MbRuKtIi z)c7T1M<3G*tne@Giy^kpzhFhx8@a8`(-&UGUyKr|-rRAgT$SEohadKHR|61yg;z32 z`s81pLOo+No>7n}xniVC7l}9&PLIFgCVK$e9LHgNwSOPP^MRo=EWn$3q$Px3Ouc1P zTV40PO>uXMySo%CQk>%M7ThWB60A7Iy+CnycPq45aVW0E9fAasyxh-!{Kj}c- zXRp1^+H=n9_K@CKfa*{=VEbLfdE&};E0*8}T%PE9sOT9jYbYY|K680|LY5i@Q2a~m zdbm27sDwqp%Fjy2NwL858;Oyhvc=Y(30>uvVPO%cdh1B|yasBxC5Qr5y!uJQg41eH zIl#Oo^$d71iq4@0G5qpWGmB;jPi#WL`+$G3(=6bmP!Jx}xF3R4g6J)Z6>mbU9;)eq zad%!xpVX?%^@+Nq4UE1u_&xdU{b-H*_yJ?Qh}E?p%gU`=x!}C}j7u=&T!5>q_~Rnx zcia@i$kXZQN))W{LE;4Yk5c#kZwMBU?DIuQ{-`F>25 zTlikA$apDyJZw?4m1n1LHb46w4$1$_F=)CIB;YHm zn0oLEDzQwG{^FOM+7sl-y_6<9ST=Sg(2ZMED$8F!6Y^QB8t`EnL%@NKk0lnu$y^fu zWeIz#uPJHT$;C92F<0iiFex?eAGA`ODG;~#WHND!@_|T9HFmrew&gh#iCqIATZyLf z-?%9GjXc(QRzG@&Zx&^MK#^REr6P1|e*-Q!D?BU&H-HW>dW)uhWkxMq17NAXmOkEV zjU`v)MlY?{6YX=tB=SN-9IL*t;ESlUUpz+xH_tzwg~%F=)y?k?D@#Il9W8RT$)*Nn z7h+Bzs(?x?N{0QNHR5;^BG7lww`iB?^H~|}_BIO!Wg_*K#YegMcW+6o4SgD(1lj_F z5_JHprEVLi-mUMhz-Q0v08iV7ZA5a${=%sARfzuP+8YJ&TE>U-GA!`RHUhGcpprrZ zE$$cLg{(IJLoFtXJx8)o%Y(>m9e9gxH- z@VI{e>os?uPW0i3QC;3!5MTipsLKyZ#NQd3k(33q{_+0sZR#=M>)D!MPBAnm>zy6#(9%Zu{rKPh0y{s$qURIhv!_ zjgo^=qjcV5!5S-z58o$J32>^a@ZXr3m`Qk9_Enil9sQdyY!6JZvQ2FXyWxz<*WYI3 z!)NRlKnfoe;@sli$7S6csIDe+WktA1oB1R}`c3rB$*?IOi#;V5GnrKs-iYcD^^fA# zrczsw9Qu0y;Y&&BepEiq!1^Ca!=K8EvvyzXCW3+YHQ7N2!O;!)W8DpL{T8`uNu(n$k7``Qa|2 zuK&x&8}5l1h@kV3^%RA?tVZ2mpx~>QliD|g>KN|r7*pjA3XZxcd=%T}gyI;i2Ys(zw2tg90+zJTnz8IJ( z;ELj6MQju5Y;X<>{Ocg-Hp^r8eN%W>Bpbt&mWgo_esNH=3OQH|qd|yg$9hUPAr;|i z?R?B6W!7!)!gvSf0(qpb7%9ZvCyg;4-ica_DIEa(A=PmFbY)d?0KZW|)wpf;r*xj! z1uv2A^niAM_cPUq!W)YRS)FKwHkdXR;Vng|VRM;DEf>>0sHNvJqqDo8BmQQN!tzwg zdB(&W)W*~xEcA{1ECC-+neO2p$pD{LOAi0w?#?Y1W5IH8(DT`&FOqwh-onhBf;0o& zA2Zdi>JyJ9Lz4$Wqf<14{saqNJ;sq!KFMd0JfgXTx8(txM}V9Gjk=jp^`9XUQ@v+j?EbI2d6@afC}V^|2p z{GbRwG$3&?#)GC=p2ni|nY9N?#(k*|*Wp|G34^8bMY>mnSays%jdvXi?1=?1)ZiV+ zBDC!$jwmkZBVtI8=Xc%GUG2C96CJq*98{9n-$Y`2adK3gAy z9e0ckvT-hntrnk|UMTUIne&6fL7y)8JJzK!@^>=w+*W)1DVH*_EJPCe=@f3p8QV&I zw8n7&R-s|n+e<0tzpK!+{7*jYPz1stIOo{_x-xekz^{0(D&sK4RC%qjM?;uS*0nmO z4lfiw2+smqJf zKlF9*8@^=Cs$&Z5E>lDAnS0(ZKvDHoMp&p#0s$yauH$zO#mc5Hmq0Sb2?`q)C3|A07W`TMr3@aLvk2OEiM^}6UfRtnkgu(x4F~2-%z7vq4 z)*dtINGil1&%amRzGyC$RkbeM_|g;ePoi$YVYTYt;%8 z9y>_!pZD+2Kqs{EhwbuVC$#<)^7)AgUM;!*h;qv&#PD=_rfp&>6Mfjid+bk0 zcSvVu8=OG+4)M&cFeFZDfU~g^*tOn;OIro*GsPv#qI(;_<1<3W1JAjWFzJqPL;DpQ zdSMIHyj)({X>pP3>Yd)rl>P_Y=dVRaV2e(wJt=%Xk=}|d>>}{Q@w!*{iXCVh$B*=sh_NRqA3;3ANw(pL3$A zf`0V$_PHI{9OPJf1cTYnb-d&}6sQszu;}_8=lqiW$X&@@R*WXU6!2hMmSzx4v1)P% zHPg+{Q1jx;v?!$WsPQ%Cl6fu}I@>$}^k{8>=tWe_HRjI)8tP zYVvy9a^~MXNdO^7FMgBDqsk5a$h*Ao<%kcGYQld)g6Gjbc{u*WrfBh-^KK*w zf|II5{P+FaR@+_JK~iFtI%B~29z=}(CL;V-}V6SSsZ6gU238~+Ly`}_C$HomPa?SpIz`!s!r z%n%W)54Dni3=cA5swsATk7Zb)BVihe!fq!>dMV55z#W@wyyFg!<--!&DD<{M^$Q!a zzako-(GCfr^{pyM|9j!P6)~L{>vR2vM?0SUJTG5rc08~_gHedC@G|%L1FL6bkM?CE z`M(bV`Fs1fv^T^i{PP1(fT-Zu|5AELZNp_~U3!t_H%pSlFA-ly2PN?(i4tN-viJu{ z$N8YArQny(5+CS06STD;d}>VI$NjD)?J^tGm&!8-Gauu8^il zoAT;vKWVor9VHR7*8$%W-@5P2!WcMxamqB=n7S5GR>vGLJa6Cb8@YO%(=Vc2=xe2RpNWx4Bcro8XONx}7Kqz5*zy39wnfj!XfQ>+Ahkhr*KrOA+5)*c;%z(TzMZ3+Qxevg;WelT^=eadw1$AyrPzeUGGg;$pU z@X=O?^|~UyqYt$Te$f)T>L!i;gC9tsU%JilG=>vrD&pJrc83t|5;Lsv5ettKF5e=H zUh{UtbH-@<(_a=T24g`bt+|J(c4Y|$-ZvBF7pBqgDUi^dx0D7u3ayp0tv`7_W2&|p z7h#uOrKtCEYIo{@v!G~OxYv@XKa_N9dZz=f5B+m#VqMK=I6b{S0aBX2TON|LnoKyu zvH;(gLQxWUi0L{Syf;Tekz+Qk(iVj`HXgc=xk?+21o5z2;Km^yfh%FghSiOmf z1t)IC<*Y?8Luj0d>Rvpzt&D=6dHgP1!jID*n^T*(0IzW(LCSJzM#S5-i}jxRU}FYq zo$I#KFy;KUsJFm3&y_e7_&I$-`5+Jv1e5 zJ_WA8uAnHLLW6~SrlZmcxOiaClt|n=^Ra}-4TE#0e9@jWbwx7@ggeVLRUvNy%vHXs znua0Cea6b4y_bN%9^Y+blj&aqXQG23jQ(#8R$0ULDXPh2tMQ=<74{^E&prb6=CG$7 z>diDcyHf-527%N|ybpLDn`34l)_hL8>J+gQ>WkH{2sReKbZRpD-m#uUk(LId*nW$l zBpss=^1;Ms{L8XyXs3|4LhjmkAxOl~1zP&Py2b6kvg)($LQ=z~voF|gMnd#H>@bqI z5_+aRBMq_@)FR7@h!*X{qOAJSDP(|O9 z=SHc|>(%R-Ul?=%Fvc8O`J+y!2A^=vdS|)c!Lf@<5)3x|vH~RwQTZFAO&birZ@ZH? z7J}0`cNrHbT#3U}YDik2)I^!$lNd0SlrSz5`XkOcmt#Sq^+~jUDU+Ljx9M;Djfe}G zig+L<6y*2y`CGroK0AYML!lIon)I_fDJi#wpT*QFhNa zK6-BacKrns6^W=0e(}D(B}}l3dp0pK3EhmZxrmjJIEhN`c*|qb*)TxJas43iifS;K z4YJu$#<*QE!dPx1ZbGA{^I_G(NVPR^?Zcw*T~e<@(K<=J*C#-Yd^L|Eh89|53Gw{FiH2Z!z9DFmR(0dPRGsU;(Ykv{LOo``u zi>K9CQh<34tNIcf|1J&7oGeL-~7uXi&;ML2atDg~T^ zcAVj~{i=_37r#evypb~*Y#7FUaLlIbDTTJmIs~n7E`NRx$P$0?{G_9!<G0it~!xzMh;##veK@*F_?xgr&0lENV*0rL0>68THwq3|Q+uWh?0 zb$7^1)$RbbnPH3E6fiEUw6>D61-A{eeiQ?m?Y)m<#0?DDitN6Hxa#M@92Bu zweernPV0~7yX^{IENrY?|8SqfRU^@O;<+n_*@GQS8%6?_9rhE?M~`3@dG3RG!cZe( z?MzK^Q+(PLV$s+F80chREqUnZ?>IAMw5FIz!6TPP)j!$|1_P;V zbvvcvi<8#s>63pBr~-fc-MDgnhGv zv4;HTUY!~W?X}BQ+{Yq98D|8^eNL?rz~BF=VEL=z?W+YV z?MeWL=z45!tpnc|n?8`sdQ}cwtT?#7T()#j0?!+21*WiwH$Pk2jP+ct>hkDs1wNWm zqfEGXZo zr^+_u?tevhq!U_aZn0&Yiu#A^%jtY}X}mh?iLxfL6CLw-Z9@`H88z-XGhm-221cA( zI=P%y#HY^PPGwtlkEYZ_IRaRA=swT{!{FX4D@c8Mc{iQugW6LpRLZoLe=dn!23-PHL8l#zo1 zQCIIkLAt@RF&c!)mBgMQXxLHZE&7J$2P&;c{Do52s|Avpx(IR1F}=$I>fm!UXB(68 z&Us5{k16f7`kf}sy40Kfc7xBI0-x_|j<-Q4?buml-j}8u0@vn0dCrbPKW;leC5Tsj zLemf4tblq!U%>liw&i&viu=3ozQD#`OS%^2oNUB0dfr=*3I;>JW;qW>=AAFhNO;O8 zn=jhj)$74gY==Ml^0Q?g;gAzDC{@0^)B_!z9$o=mAx9mR>L#mag>JfM_iN|Wz8ju} z`M$dfc8!~{k1UCKJ2Y+c?gYHDfp-iW#IRwm{%f&cHoWdZg%QthA&~yAVABje38o|B&v+fVfTQy|&g zztb%6L(HSNZl-+D`T6v*Z-uopvB4WoMErkD;TZ9fW zwb(_|7Xv!hS7aS(gvKH$H^LKt*Gy%aGqqx*ITh;Di=j`;O7DOD7I^(1fv~oD*zGBD z@3ZgrwH_N6;Z+Ww%gsAR8EgddsN9|jg)_BNx2U*)oG!nVUSrdC{FennA%j-Hi-@MVwKh2Pd`Ac2~|LxVEayJupFAo#DM*ZX7jx2NqU2(Gb8w zOK9lNO=RDz=Nc+Tpk3D}SMC3t@~z;SQDCM#4eA-AjAui}dz3tjOW7{R;&s{@=F)vL z7IM7k#CcD)*63T)s`1bPlJHW1%!3=4FXD^xPoj#xoaJ*#5YL*e@};0YSTc|=7tgGJ z=_49P7}`^G2%<`50HebM0XZ1YbiJlYF5hP-D(U=P7rU?Ox6x>Jn9Aqa8~QKt`8UnY zPL%tTyhWI{Br_Nf^)*ECTV##Yr&AQNBV1A+w_f1rVB|Ux)%jg5TmWE?Rg7*m1)Vdc z7Vf8RA=kMLQuwQ0*e8G{v81`asC!+*ZJzJR@X9qdSJ*-0a;GC^8g!DUtdzRE7m}%) z*07#wY>rYIEUe9Np=tps90dFfFE!ngF0R5y!^UeLrp@?gg<@i(;1ypd*z(|1H|j5+ zAQH?HUKtTf-_Zx27bVWXzKD0KTY8V9MlRD>yD!^x7|)ehLO^mru*>m+*ihubPhLf} zN@0jG2gW}0ZLCWfOHRzxP!9#7`5@XNc;dxkJjd0xr1uWhCVpm6l#RVF-NCp@_$*=a)AeyZz**kSl9m++&8pJB`;0cqEyrzXL zzu+a;ufE9_DPs;_Z~C)KI;;-~4>L%LENksbI@41atFwdWhDL-cMLSY5DJj>Z4rmDs#j`jnJA_qt&yBed> zbe^p=Z6G+KLRBWp{PXuG$^K0DY=nfsQTs zOJ5aJL2gSwvGG7%K&IeNXu|(^7h5mzhcS zp}4UrpgQ)m+)%9JT z{Y(k@#C7qG!orLhBGXlzhB!h7I0(I!sa2FnwL~B6{>UO-)7!;ibVYieb{>Ww1_v^{ zv;wLd386+6KN$*cmvtESY+HykB7iP#+r9P5s^cR$$qFa@K!mM@_Y)d@??x6yxcLbl zfkS|&22d^MSfn>n24_Z5b81sj&^Ayv>GZLO`H{K-ppfhEbvb7Cn?MQ_+7GdFM#{ZM zhr6v%;UO^K&8>jb!Fp$I%4-;ri2v_43mx@U+6U==TY~_t@8&z+D&^l2<;Be+~#+s zZYH*US{+sBC)`t#fAxCuq)YGyyZ{&x4LI32{bLWpT`O==@QX5OwX58Jx7C+YIi6L2 zM72!D&bJTNtd*R${)qVFVfGw6s)MK5j%eBazR8kuXI`)>=wjNDO9;NYB61+;fwpfp zY{qqq+Pkg08d;`BOHRQ9@8}3Okgx2o6eoLm9 z(E7S61U2Q~4gK?-r6KL=lb_?Hwutz-lxga{k9G5OS03vC)E{MQmEOV+S!_&n-OS$A zhc~5}SDQ$=R(9xenaW8{T$GvjQP3Gj=IKN)G6l2h=B3Ncm(cN?I`nRkWfcz>wW^`U z(1aRfn4;ljS(>Eq@8CV9E^{lyY0~cTq4W?B)2=pOKcupK6_A=~cqn(@Mu84jeYYji zMXdLDM`4eC)tXg*8@aBWtm9$)evpZ1hPXuXA`^uX1uw_@xZ%Bu1q-4B!;rLWKXH0O zMhr2jFQ*OHfs~GNHJmpUY9V@-Nhr;qVTmEdsj>OPDdxlXxDU{H4g8z2h!4ga3uuG- zuq7)0bXK`F2A+^%4XwJ=m!!o>Hgl#*G1F81!4VUEajl;3G<0S@QaS?#S?2rl2^5J^ z=uXp`t2(UV7Icb~cs6taY?`9;WHc{A)q2%_)w!(%b2K<;`38X5) zWz7{I%kP}A_jQoC7R{X)6zY)-Y9Qf-r3$$S%)df1DxONsGpbn? z+_)426*js3P)7=I_v!QSzxXm30(Fu|*i-K{Pr}n+w#gh)F!BnT7J=ChR7xJf_Q z%5o@s$Y5&o8n@icU_6DL`oaY(!`k#>NDYxU+HA+=la{+f$y;;8)!i}z60glj8g3MF z|9O^^*cLcL7g1NfuJZ@+QlqqvMShfV7zbB$87Tolnrk`n?B^1t2>#WBC1i(9=0$Oc zJ(eUfxdcX&i&<|>=GG+TF_%9VbOPG`1~dSA zMg*F#?foYq|Lk4U<=kH&7tz| zqBSn(A}XQ?c%-i>g!;FnQB{>}0f2w7?eMMPcQ1U)3`4oB(XX9XUBxIu(_Qih{Sz{pG_;;{itAeEyDSM^m?f&Kn z!T;1vcYuD@CAGCxjr2ak+k(cv+%>hPQ%vFRyigma^`u(Z>AJfAs}*ejEfxZHQVUaC zd^(JljZNrk9yrM>ch#)@8sad$vNltQecdmA>?Un+Y2LOKt&Qx7LTB`IfPg0MP@Obg z9?&;0qcx$kVn!BIFUeDdZ?eS*`Wh_jqUDbtzaphQLzjKe-O62+=pXs5|K= z$;P6~^iorpL06K&UgNUVy;vlry^qJ&abWo-wrw_0Dy{W|F&3UwgW|`zxc@czW7WdU zszY|WMsLzU9zWy2spe>ok7#2I5k)Wu{LN7#^5v~pX=c@rWc}5VPMa~dlbDg8le8Eo zWn9JJf73D%se5s6eb>{A@SNZ;i8i%=RT@{sc#(2{V$AivUvu|Bk;NWMx88&FBkLwK zMM1T6-d_$lYveVpYjtv8s?kWa=@_y)H9@Vx?BdWxZ}PU02ghQU$0Q;nt=eWy}R8QU4$d5 zvL0(d@fwv3V&_u7NC~?Jr-_SJkKE-KsyLOi8)(|>y4n!?$oAQzpaPN1oo2PvJAMCa z_v!o5cx;H)rURs!-qYW7f*7AAF&tZe2Csk!(<=>pV*1#0%b>^vl1O-$Y^4IPofxEo z7@brddUELW-Y)|SMqL~dr(SQ(;Xlv91oQu4mr$2&u;qExd)35a!-Iy%-$}P+s3Rr`)lL}(o;F*Ztb4{x%k~*lWkXLz)8_gwi zDPF~s=Txz)2W#fH${Fg9NnE^{iUeRwzHN`db=U{hAKh1?Z=3z!8-!(p{KQIi0TD$& zwNK9%R09oE52o_JopqUKOWa||E?eLz)quU$jcl*WiOfh1QxiZSR}T^+ESh6*;c%%@ z0JxnWzm-zZY~iuS#alPRPgO}Wule5blsS-VRhgB1F@YW^Rwkk>k&>ljO2#mJ^V@n+ zjW|f6<+5ftN7He{*W)1^%Udde4uY0#3!EPNX16D5h=o_?0{gXKtLU}WJ}H#l7r1A= zmtWk>%l5~^U+FJl>cMZDIC$(uXX*T&Sylhd7WYj33sxZjJ$a{l)*smS=IL(2pLz@) zpDWBhH&u>|!cL{|eod~)8rO^x6WKUPR%Qqc+j>@weqX4>rU+ueY5-L}j}9c$eI#U> zf#wo+2Zk^{rmrIbqKYSLFZ)T)jxZg73U$8UHI(4(hv$vC*wYpx-Q zavl7OuHTA=xy0B}{9Iv-DRfNyzFG)zf37F0C91+bys9`8<6(wr8RjyuC2DV-rV|(Kk-)QB}GiA$@EU3=8^px*ov3ZxsjuG291vVGQJIf`wkm`r@+BjIOU-aJZJrH zIFrJ`?bXy8KZvF)JW+!MQGJB*ADX$@5A8Lb(2S$PQyf$d#Fg($RtV6O@w|gOm9rU~ zB;Y4)gU7Y@a80^odk8_?OPP;Af^8)rR!pRIB@OQLUJ#K&K`;<4g^=VnvOn_q+@Wt| zqEcl;+!gCqbYf zm8tmMG#SCjA%m5ZQ*qErh7+quqog1sQPwbGjSv()l7yXH)Q3Mwbfld*xbS>{Y!XXc za!_#evYMLv%p;M^qWID)4a+OqHnfO#7W&y;+tRbNIGOnVL#QGx>hn9Qp<_+YKqcllTUtfHjf|oC8n54cygtDJ;9>)RXQbQsRb1Bj5%( zaVN-12^U~QhUe~k|AKMfW3W`w_0CBz(VG5iP1Zb%h4c?=e5SG&Xw@SdeR{!iO6fHp zC{6dP#Mh%zbwh3|NP_AcabiFi27gmNN?$JlAKs9MTg--2;A%3%oUpw78*ZXv|@?&lQNIv8hDbXRJ1X{E@w{=Y_Q+vlDZd zASVBYJU@HO@kV{z+iG2Z5*3?2ksv_|p8S)dOr9Objw2ljv09xXioQ7+KfOSrLsrDs zWdl%8X^hDDN1bSkHd)Tpo;ICVqo*FPDx>%UGqaRUNRTUgLvC~dlSuRUKMmh3O|VF6 zJe5fo<<`E~*@lJwqFdturvN3~X(OH#+XL-}m3WO>Yzh)`+$rp25a8xCag+4Fm?o!g zkgs<>aGGKGjQx(!IsNySsE6TDR>ZF6k$`z9dwE2^nuE|nw_5H&IR#dre~b#?Yi|Jh zxNgH0(0bYQ-&a0eyMR$JLMy!3e-@e1XX2H@|2?r^jWF;pe-~5d7nk+cgE6Xxq&v$$ z_?rgz4G+_9sXu7PQ!J+|sUeB!pYgC!c0)6Kv>I_B(_mG&AHLA#LUOnggAUEThFJAD zmC5@6{4Y6bfl{r}srE6{s2QKol!-Z=S84MFbcbA_ zpc-WxA3zM{1FIbm^eKDjxbUm{6&49cECgCEcJ;a2PDK((yCP9zm!L%8!icaA#&)Nb zeV077&p%$`cJ*~!O28=Va~0~+P@WP?w?P)6T^128ciQWeng@;wL0s$g3oDDB{s z`_EW0pqIb(j<508>M5LiZ>sZ}H>U2~6Ae6PZ0e{Kc!qRiST_^tt3{l>`mEzcV9c*) z$6XXkPO3l7bGf;A1lF1$?~G3CV36FHcJ+@aXkb^hbD6TS{r_LX-f;CsYu$s6WP<7c z0wKz%FT|}sj)w&~C;#qxlgyG7Z`Hh8%J_~$2YF}W#WfESf5tJyXQTxj3V9wYSvAwc z-yX;7+2YPyqgirRSK`-EZ&idHl$O&guBpvrDrON5BZ8*JH_MpPqgCSOMS0c5ekWa? zFtQT4_~M@`)J!9aMa(zkg7YIjui8ru_f|IH9Q6){`Bh~QwKYvQacW^j$kbPq$}sa* zwHy2k30TX2xQ)iKde_ZD(=4(FL4G0VQ!2}-x40`_emE=KlzwUA!?t3~0^*A}m>F1Z z>jc@f^oL}OhqJf7H+9L^%R}sNY_>_Im>)m%uC0a!F67}tn?f|0h6lzE-?nEAa9aRo z5FY&Tv5j1vgGjfn|MOQY&U5qeTTHSPbb0V?g5x(H_Du#EgO|cPrTaOdK8518%}0qn z19U<>_4`P^?*CJ;0&L|rfI;5Z)E9NC zt+3OPxZ{~azhl;60^DV{Raj1mKc+?iZp?il_dW~{f)SdQPveZp{sm^@?(IWtY1;Oduni6T?%N!8ErkrG%>7?G+U-^wj2ECy)d?i%;MvY zm3i`(vGcsHftKrBdu`-iO7g6idaw<8msL{e1zT7fdjYP(Vsu!I_PuTP0e%DzQq3<^ z)je8Yhu=^fX8*-??!CsJsha@zp#%?G)GN@uh(On_UhW4-;U?x9-dA!4`kMVZ9!2ZE9rE0e;W1|g35Uk#P|sDQ+<9?(<+8UBlr9Mcv^!Eq`QX7AL%VmTE1PNJC~?6 zuuF&gm|`gTf~?y6dGuwPC!rlJbY{3i;Zo?SQQJQC8F9FW%{BjjBxT9iYx1MQb}ENJ zr&L6#lt%qV=jbxiv|lSv9G%ae*~Vyy!K@4Ol$I?Ei&X8ZdH}Et_BKX2t{)#Y}{N-2EOWWCp4>G-07ugpIf$zQN z<>S%1{jJYz`uz4Q*p|5XWP%tKt_CKzuzgQggb{{Zhrr0 zbs~I45`psRTO~;l{Q`fO_1$n>d?x-+5s3jI3x>d`o+?1=o34&Uvm%_>a^Mp+cZoyD zRp31^<8q)0DhwM6n<9FIjfunHpZuWX<^VC+&*v3}JN3_~$ul$WxoGLSbqG9l7Y3@O zbt_iLgrQgosjX?J;?!R5?&JY3psig4l_;~{dR)NN2c|7MSHiRg*W3`>ZunWP4I{b#UcTG^0b=j2fX2X9s3Qz7 zP4<&psa}b|1~ITsQ|tv~`T}55{k_l#g;C$BdmsNiSbehCHpEV2`hR?hsj_<$k|S!| zn6sIV1!{MQ>O&=u-5xY;9#As_Wgmhs9B_UBOBSr&<=0z(VHWOXn%hp421=P@oQ=pS z*a=|u00I~Xfh$CZ$_f(=-zXg}pPMyP5A|EOo=M`qqv}`2`vHkO0>96H_+g=vwiu^8 zp`p;Ub$bAJY^Op8FC)jQ6N#=@7NBL|AMynDhRZCC>D}lRxqBA~?c46Q9&ZMNciayF zr;p3bjkaAk4T&VCOA=;XX5Gwlo~fDq_0lDu>kqu~;e4=bqo9-a&Ut&@=auW5A_4?ki;|Bw+aXD zIbGhS(WI09L_63vcxJ@QcE-w1q80Meu+DPBCkMU5N(`Df-M}e;7=O!>qZ_cp_cNAG zhlvo-jQ>A~Gg^ft(j9k|%&HfogPu2JGV9hGH#U)R$@8KBh}mf3{LQa{k!YmsFcNyV z!4c$n-+nvHs9Gv5^HWP~nIo+O!H|O~iG<#R^*eopSmj1%tHQC)yimrLn}3d2+)b)7 z7~KT_D$-(Qv%A1{sV$yW{W3NqWI7xe>b8&wtle&E8$Sd@zMYE(3-C-&d!(5k>!Mzb zTOJEzV0bWk=r4$bIBXQ89xssvhra^rgpERgiRA<2|6W_dWn-6#28M^|gTeKcQBSq^ ziW+bFvP?;HWQJR&fEmUPEwq`z6>s3ADDy5OB;D(B5>NY=$UeZuj+JWoCqdX2mg$xIF&Q{ zZJhgAPs+voKf;;k)z^-Tr6lO_JdLtDCkK7KhY-h@;@ZqCFq@qnG>{|4abVt;(Vesg z^)NwpwH5#^`3<_q*T0pEynt6Lh^i>;t%-frH-nLRLf!;>H{5%Gh5Uj4E%tw_o*M^^ zKby%NfYN9-W9Ax9Ay<0Wr_13L47-wLZM7`e{IWU90hpb%^YJ0}WvQUg>`-7)>99&3 zu3&BADI5n|`qdDd{JF;c5ZTy5MFWjU(Vr(ET`RD~uF`R3c^eJ*aRDJC%iJ#vU;P;$ zR!)TerY99*ItnR7b>7E|q`>QER8Zl8{?;9o`h3$D`QOh*`Au@^F{WOCsabWRc;J*g|C2omH$9b; zu55S0_^BXxk8i(Y5MQFJ0SZYwMqM3U(J%s87lq44!^?kqI)A+B zo8qhc$Nss|UP&98H4!VEw9=!*;_+%Zp4QZ=u6Fy#CSh1?)LMc3#@%Nb3OragjTf@7 zvwRCTHpF@%zx5_OYDS{sFWE^>b$fnV7hfSgA?cT7EpyB4AqR5rWeRh|zGguKuf+Q1 zBjY>eNq{J|F*>FpFBYdOsPVk1g-QDK*JUA9(D06uLTMNR2ZF&!GJ0pt_A5a5?JB!4 zq3_G5vc0$AY0}-cx`sp;guDLD-8v~rQx6NJy9Bz!;$HyT=fEL(MOy(!)Y5w!rYKaH zd5)~)HWQ$Eg!Da?i0gNy`xqqU>P~jGXR>U1+BPk=UlU(#iHdH;JKh?r9~{0I=SXaJ zTIJZ749V5XxC4`HEqG_XK%7>39Hy~~yRAlIiQbj|Tv!+pQF^!VPxR}XWt7+?)wRdQ zh<3_v_n#TXhl$%U>*HeSn9weWM}E}zk4>N(ul0Bqv{Y26VbXX-eTGZzCVf>skXBgH zP%7{~YfWO^nhO2ctN_sqA7$n&qy4*)~vdVEE-< z`;Xq->{iT)_Z<;-ysCthm10PpozG6LQZWl;XO|J;{%eAa4B|GGP@VokQa&>-ZW#e3 zLiC4EwQJxABSno^jUH7duah;tf z81E=;S=ldx^iriY-EUMJRkX6c)Q~}y&$5d3bS!;09z}a@tIrxUQmt0Lb!}j;xl-3e zR%NDfud_}kQ&n~B?MdfB|DgB;TtNlxE=@#Fe(qoF?MBqKx9cThPce}7btk~82M4;> zM4UF8%fw%XmESy!!9#@L6J{3Ixa#-BI!`Gh3 z-Q-m22uNQw?=Q5T1po8B4cNEy{TIIGVX15vO&Hb{_??9pqR+?8K|3kNX+ie^lVF(d z`@T-_@58GMPF5zu;@$$ij^r(65MNrBgc}6j%F){UV|{oY=VV)RG5*U4JMn65mupd+ zHsRVOM5t8ul&7(~pRyO^#(-ipaxi#_dIdiHjorId{;1$w61`|`Fj`Y9|@O9zl7+cm>rl(qAZJWAgrwxV!I;if{_gZ0e3W#~_Q zl#dMr$tL9|Jr;F-s({;X)Jcset%Z<&Ixv;+R1qc}8Q?)zpP0?7p*-B7oQdxVI4FV5 z6^yeYiR$(Zy`)urRp4Z!E8!fvFIoeO-}q_EJ}ou-SG_^|YJBttJbC)pr@-Gsa&xU8R4!{1T`$A(BxCN!!URh1^;xyqO~=Jd4nIHeZ9*D_MVvwfIVOLsh0dnBX?XE?jT+ieqHISW~)`itgbgOR=aKbM8~NaO`O-frxm!D*z|Cj1#8?J zJ{%vW02`YcTjVu#L{qk3m>V0Ieq%s=5Js%MI)-UEYZD#`bu_%6YJ35Z9r`Hiqyh$Q zNPWZ@K6>nZ*{`c?lpB`B9iJXpf`cksD5VbycOP{ObZA;Nd=6k*sVDWWqU@Y0=+u2* z!tnbu$Hk|M z89QV|#2OX*#Dg!U znO4h8d*Mi6i0>Gx#dvUNv_luGgpA9)0lRy3?xB3ZIGwzTscBuCN$rm-ck(D9Bq6NQy_n-cy#>K8Jn=XYtS#kurj`Am7IN-qQDGXv!Xu%-|mtOJ7n zXqk-5cQAfqI%wVM3*aX#2Ida^-}?S{4gr54?L8m;0zy6I726kMx(T_|I=b((c{ua} z2+SDYLWS6Sf56A4hB7PC(#m-_maE96Nk-d$Q4=~HI#ZelD-WCD=Bax;qWu4XWM}2T-~@3U8hCHwpY{I1ez>DL z8ew7S=>0C{+CwnARXEE`_dN54^of8MI|~wlVFey9DVyh}<6(IIRQ_rRSH#i%lrGL> zNL%3USOu^_4yUR-E+ZZ{g3G6ymEpTp` zt+eq2vvRP7YTl;g&-o??iwAzo_uF&#f$n(rzUetvMOcStoy2*nM*9`rHJd&lWDXII z%ZSLztFv453*Sk~lJDzCAq(gVh@tq-_hRZBw2=qDE}X6$JZ~?asn-rTtiVuVsKAT=kFCFq zih_H;$6>k~DM7khQV<+ErAq`EK#=Z`5Qc7~r9ry8%b~lZJEWwfYZzw!!~OYv*YoCC z^MV&F7H6Hg&c1qYvqOyIAu`~7FZMt%BI)ayEj@^9g)`8=83_KhV-6vE7 zCfTt=+|ndrng5PguD}OP2+54y8`g&{e61hkyG6bpj5AMMouhZw{=@Mj1zuC+1i-+A z{KiAAkW{Kc2&hBZM;VVS2yZRaK&=2u%u-IN?n=~pIoC7ZvnnF>{d&;lU+ zMgPPv)a?kDztnnstO(cRV^M2>hKX>Q&45y}aaHB^;D=4bBECK0N)~8SqN6|mal|r{ zAj)K4SACsPhgvhC~-ZJQ}Fw+Uw%kLqy5aB&`48gXz@g7KDNuY)IT= zrTe?YiDn0(M~-NTn%kFT`Aw& z2Z|E}5wc?a=fJ8KN6fc7qHIQ~ywo}x1b$`BZ1BsNF6aui$X#ZeK2YcERxU%-$qWFn z0G-w7M)bc=(w)$zG#=m=e+R$B&HOfX=5TaP^_@DgcKK_rWTo4InnU-`JYK4qmb=5! z4d@^_BE?mRO0x~+_saoYSU&dk#UWL+WzZ%YSO8WXtll0C=N9~$rP;^B@V^L4E~l|+ z^`jO3b6=stxmROzK2`C*g!;;m=m4JK0gGpjTUaO-hxbF6{$CS;ZULD%r*bG(88mqF zb-o(x^~%GjC-D3Erlwhh<_8MYTsbT7eC9c{hR?DIyUFLr(zG>nVr(T9cbP2i_|NK(9ssA;i@(8nkLzjEfX{jHrFyYwTkcdklK4i8&e zcLNKNe~9X64B-9D4skOzuT!ZU9>pV_t8&2CE4qMS32@{N^Iq`}{^~+OLd(8QT1)!3VWV9NZ8rNiW z);hLQ)79K$ySnl&`zBIQt3R0|WJJr@*dk8S9WcW-Rtev_I+0wdt}nN(cr`Ro$Ud+! zEU@B&AkJN4VG3*SO0g`x8g7`#)Y+_Sz$7fG0ivQPSIxNs|Gm%p{%WyB%nPYpL8h2) z(D?CAY@U1)c=A3SrBJ0>PW<327#=KTrKf7D9wdRj6f`m1jh~%EGR!45S{AN-~CN=0?WBoPJ?JKiA;ibla-}&{KI52a@wlQ$`M6qA2;OsDXo58^`Kb&b~UO0QQJ1x z{~}cKYJ5{`s_f!**XcfW<$U@)6y=GW|aej^A+P&L;5TNRQ2Mvo)Imr;!;Qa-khIZHQy#Y?rym--&ryd=&j{tga z&sW&n8U-?>DO0N0oq>%EHkmT=h4g;=EiOl?E>p(La{E8PP9T4`N}kdztD;CXlAHWx z*M2!u!XkyY-skU>dkdR?jV{=&4-lNZo6Df6DVkGa|@7T&;V=SaCKbg>=YdQF` z7M-0aW95ZrT7Ct^6yj||rC?qH$44piXyXbu>!dOzd=R63hl58k6X}vPQW)kOxmaJm z&9}-lpcK633oQ3ZBBf+`{I^HKQr#>k%bdcbFyc#RI!=Z#A}uyCWQCCjS&a6Vv1w$V zhcp^j9l=AFxK|FrHw!IXd>zvwXERi9c$!j8{3pUK|7uKWWZh}bO_^Sm#8 zN&V)8(U-P~YWeO>!6Jv}LK$0oc2e8bz9BfU?*zdCQ}A7UHC06I<=2#nK&=_GDt!?Y zg%Qa+*rcAN$ZBOp>?^;rs?EG$J#LN`jdu^d;*PC>6;m%}au$o z!_1gIR%bPhSTDg_FMv_2SLCXOP1J=!w%rkw@JfTU%xn+4B3@Q}d}0YPIQy~0&r*Zj zf#;WTY#nUy|24%CKj_G}ucuEco(sh=R0Sw*EY~Q$Pvn!2M+so(AxZ2+@l5Y442IyI zu((kIPeOWlBgfi9SuN!)FukdmRU+O;U;=XlELkI!5B98!eb~D!B`aX(xi3b$wWk`~ z>yt-;KAixIr;wIdEwIE)@(ZCBU-<8-eQqdPN-DWh0-6yEmg{}dLg{1PZw7h}+}BvP zX0}3n_BKWitu-Fjb!`q26N$#1spA!C4!QnMVG|j51lm%$IBM5az zSiVFus|rnw+3l|?cbM26Hy+svd!$GYuq4%9Y=LQHz?910eb*F5Zekc>61n(#(kgVF zYV#N>^a?o0Aa4Nbg8knv|4=osXBWMOS!Q)Cb?#CuIA2g5F*w z@7VKtzP;4ik@atRDUIJFASdN?R94KPj36#?wKWXnBaq%^Mj+5yB zrSQZ;hgv>R+NPV=%amp(&C~6xg^WiU+oW4ssG?(>c3^o5TKcX3%epT6+l-6INfwOX z1R-?=`ZgC9M|l|LI3@n1`?lrGMnuEJAwVPg2aD%=$f~AbI~8h6=)&W;$->#S34hPg z$v`Tw<%*ZaCYPy7pcVvt>$GK9`U(5sM^E8cOX^0NZ5Jk4M>{) z4>(|}Li!CJX5;xLn!_JRYSYG}#HbJXZZD`f4T*daK7R_OT7+9LTB)73!tDs$25x{- z5yt?{h)23(IRPGooYvc&+pb%~qhsZ#ZnUM@KwjeNGyL=>L1DL96? z>RaCV;x%PR!(uCw@GoBYzSH-NtA6<9VkNXFZB2s zE+DaVeiA6w-FQlK1NJJ_}CwWF=ui2!%H% zsfTC@d=OD7U*Ee-a}iyRO>%6yL3q=h`_~IiW2|0PW%3SIZA~`oYqXEKC6!a%c?`+rb*ey)t3VfN;9eob}ggrQfmp(!S9 zZOuBOS)%K-!lWfYGB~!|0XjCZaZGp4KCS<(VWeyFZ-yZVm}dQsq;zE0NM63J3-$At z+pHP~W%*Fmd~V~$;eS74a4w&B3+?;Qr72xMMV%&foMzauEv+m8+F(aql~!^8_)Ro_ zn^Asj;cl?X@$FH6+W@HLs%w~pB3hNRAqux!Od3lAsV0FcYF)2t;lG-Pc;9{$6H)1> zAM3L|?R>a8%?qb%K~SsAeHYue?zsH3-8n=`=IvIOPC}&EI`d|S?%aJM3(lp+F16bF zoEDl*kSIK>zS)gG0iLUN^&)evu>L2F5*fg(D8>`<^Hfm(4cj!Z;t0A(ua)@cqmM!C zrb{UEX11miIcvG-yMTO%T%MTLAdjXRj9aHZFIqXH@dF5N(1f_!*(A!un+jTNIE2mm6wwPBLO5&ZlgnyO)@0rzbUh6&l5t{YL-P-}2~SQ% z!_0VKUX#ux&bnM}gJ=fB*VU9nRDM-MI}was$tVTsYEp#Q|1{UxU}&=T*jZ^>jmyJh8N^*)rWE6JgT zkl7ka*1bJ4&-=XarTe;qPUF~i`}zA$>;4O1(g7$9)U9he4<&V{Mi6w~*gbcN{*M}i z`YspI8M;z@gZ`K|j-7wfNA)OIZzw|kL^=fqINoF4b@yhD?~QS57)I=OIQ2=-xo|7z z{R`)M3lvORqz04RTU-V-s!xkuCiXTYC{3v(t6f-8#ET5gq6rZua*h_b(nA`wBexc=_ieiaFehU~4IQymcvp!1v-i1GDuQ`zvSE(B?5(M8M@+Sz``G9_*= z2hys@NtK*x8ha)G`YynswlY}=U?8X5ZxxDKISr{9y@FxdBytw&h*vkIylNS)Gnoth$TPkF-wHs^$&6WX@7Zix7Q&p3=+-*wI z^>nq|gR4beTaB(nE!#1LKU6*7LeTuLSYz9yW9tJ!2Xe8-L z)~PffAz3}P>R9Grdv$z@qH0ZOk|y9)v|G5?aQTFFko#3M7=9F9jTC%dyEWf92~G67xn zPckz;)O(q^L2WXo5Gf*0Y-etQ7Mmc|c}{7Xzni1Ljm$$HMd3ytKjD+j%eo!RfGwa` zR=LYiAyp$iqMMbvj&&oOSHWVQw)-Lb3EVQc8jk3d1!Xa=5bgw}rfCmsaXIu@E58bp zccXXi!c@{!B9#$C$Hff31F|H_q~8Fg=cPYs*AVSUQ>*jZzBA|b(*KiNR|?roQmy;jj8=uPnU+O*?#H2O3X z9XBD1L({IdKzoPQmQ}rdY;YT;6{jr5PT`ZSBAp963H_c|SfCbTWFkU1&|}eXh}GCP z#pA45%u{_=doiA!qe(H>YZQY25~U!IMdUqX`H&4hYLa`)YvPWvY~c1~^_}lO8wAZu z_Jo4D9KZeV{Qlyp`|0nJ%C~@{I<5M+bxvxQNp_o1xe*V~p-*>t3$r*1Bf<-)H))rb zDrqnHD=e%Oc91$Ta{)87pg-8F)&;Q+lp=b~>pon%E;zqmRtMsNIt;_K%K8*Ry84^kb?XJT;uiN`1jn#;R z)Kzm`Z(d4qtiDFJ-?majd~FO z^T;6U#>@3H9Dbv6W!L4IpZO@$hI#8cT#ja}jdxBTDN(uM%psyyu4@ve%~?02U0Y(U zv>T4pVvr9dx--dtrk}{-(7iCOD2G|{%NnDjHbu$kTJG~bjMIAAP~ycD@nZkQ^9LIjqyCVy4FF<))E;&WJ>@DF~WfW90w7bV22^e6HqHB7sUL#Al|3PH)w6!+NFG3tQK zv~!y|Xn0rE-mXY4T;*+8<`;Y#uQPe#a0Z4akU~UbI|gr;AFV{_m`WzhC}XI;7O1uR z)puM;k`DH4!Ah1FamnA#mR43SVGvN6X>`{jh9D6XL>7<#nlk;wd~>q>{yF>ZREbvc za0=}V8JhuCy?qB!(|Tpa4>}W`Rp-Qm&4FvE9j_M=OuNh^EML2)t!Im4*OiQX>^qux z6a-l4_9F&aC|LW0LUF660?cmrjdbj5ByVdU9jW8A7I0GO6RCHm_}hJ!eh+mVO{#eH zE<5okLp<|#(pSWK_#yeR=oHV*awf#CE=eaKoMsJsHivvd)qZsD>-ra!MK#f$Vij~x z$9N0d{EIu=GRXx(=Eh=da=T&3;@q(l0+sH-h=Mu1SOO#ujYbs5<48jZ3`WBBdSu~G zm$Ip}I}cnrfKa9j=)+kKI9n_6eaU#1HUoYG_qIWyi+XK|AphHgsO_zrFxO7@BEoTk z+#z#&q8h#n4wu}0f`M?zERc$r>54Ub?Z}yN}W}Qs+|?dlQE~_6~S3HLt1UNHYHXiZMkxq zZ7$7~tAOdFU`S@PWD<*Tv_oH0OY>9nk(C?8)HS%}-Gd++h|H4AOY+rkQN z0sQd{)-dqeYPo@Y>;5a32R(JDsG@oPllXDIwrlz6lG|HK@-~s3F2a*yBd3?A@>-0ZOfUErBo42@x9s6DO?Hl$7Oj1lfr+=07_b%x0}F! zfi(&;vgM*m$;QHytef5~Rr4>YS^DpmzSwbvGb9XoDd)>=c#d+Ve%NdX|3iDPwrWuj zt64y0N9{B>g5Rt^UCmYf5|J)mSPeaXj+0#B?O(4`xVsT8SuuQ?>QPs1O6k@V`zi%^ zdwGtk1vZRQ%u+moV3P*EB#|p6zZu$4fj9UEqfMG~@$l3qsUKVElxi&IEioMWu5c{W z?La0QAI5Y*P1N!29CF$RS1k5Rgg}iNxXaH*h}{^zQxu4VxP>OT3Wwr zE-vc)j_jbu)Z#B#Q2Yf2B{eM-Nyr)z*gUyXYL_F-Yfq2LJeOTGFsvfs$Y8a9E@4Gg zvf2xDry-F^uQyX{|7rDe!z{yzkFfcj+3<+>^20rmG;=I2SAapxpK(cO|s<-TkGv?^Oev)~B-OPvUfTZ|z+RdCHA< zX}xGT9lLetZC~C;6F1`NB?+9wy$`3Ffv=g}E?ebxbm4is+XO$w85i2^zWQ?U$CZ-k zW`QO$>A1Rm98s^SZl>gjrlC}k9|#rMZ;$k(#?Vn!8MPlt#Ic(%Z)yVi@}$Xoh!$MQTPVZ3eH**A+(mN7H%beS^g1CCzF#N&|9A=BC)3xo?rM@_cpcPHk zl@zDbm8K*tVr~A0Dh0e3bFoJ~=?%470zdDZ>}gMJ*ULjCFH*rP-W0|bH-%)g3L?7w zZxWH0#oP{BdkmL-+x`aSd7k_Z?~&>W71*;Yzh@@-x$yJhV7C9Y#A&Md@VwF$VBw4_ zwmW_+1;v;chD?Bhju>Yg>y2!Mz0tRxfi6t}j%0Omz~+c~Vq$p4cUa^VV&qZS16C*c zSjXcQ@Ai_@d-VUeL`q*HirvS=Bu1j-57&FAY^tQq=tk^b9L`8m90eGi$=IH;9oZnxu;Ho zjtB8_PW@OjfgBYTd0Z2fo;I-_pS~)-HLb|g*ATlIt93*9EIrLN95^CVEN-8|T?Tvk zMb0SIov05($-7Ngqb(?9$m>^e0ND7?s9G(vq5{ybt4r8{6GKspFn>pT>i7NFz`q}v z4qyBpv&?+YenqSP80L-EO}8NO9KR85_uaa2ChWWzq+Z75gOYaeH7OPD{x)bEn&_Dv zHt~Fr@00AJq%!}I(#!2lt(j_p!5>K>XCOEAU6j0g*e^9FzFB-Ji3S% zV=1>}BrL~f<2RkEHO&Zx=XnI*(d54@_kx?X<_;bc?SEmq!hRMSa(Gx}n=!P4v}8GK z#))!7Ku#lp#xAR=oVkYtKS`@<1nv5Z4*Py*t7)b7y(>O&J+6X@86Oy09tR#x`jqga z%E+h%p;ea(^iF}zFUPx0DNOvu(@i4pW3t0AAzOVSgkjf+wQNV-bK!II&qy)(g(Ryk zR%b3x2l{EEue{P@SbH%`AX>PeR=Yn5xY?{>&4B-JDL{&T5`b*hv;xZXfj<|`AGmq} zy+GXn&%ecQ{)3~t&;4M$0y$DeC1{O7yfu~%mvVw~7ItN_LmT>hCO=Q_SiC_I`YNNc z$$BevBne4@7Hxd`X9zM0V$VyD_u0HTWO;ic&))mrRX}Mz~E9dJH0pGO`bM6zp$zzo!!645|Rq zQF%!682QQ0TTxk#l7270xirq`N0y1wTZLaccM3pdjpkm)p13kw<7gU`)f)gosKS|d z4A%$rY6(`WMs_@_mP4=j7gh*XS(FkZwu7yxdTJ<9ixy2lNow6ij*Gn64m177 zC$XX>%X2nf$z|oeyvkgwm1C?+Ld$)UQ*$HnWss@7^&;PoDLwD4zwmWkii>EFe;z9s zM~DIoGAom>Zi6L|Wy@1Mp*7!|RTc5}o){$#UaY)5^Rp1#@S`YJEjw+MCA8C9$f_5b zM*gL{g%|bBJ~)EmHnjw9OSk>S>-MCr83uW!d1bhXc=)Knb%JZ_E-e$HExeyZ6jaa# zi2^vdx$DuR$yWqHzLQjM_LMRj<$5hw|E=3&>c+ut<5JmMXpMM{dghk9k8JzDC9JFn z*taJytuAqS@6C&zBxwL-Lrcgh8-GsX+qYzYt;ZMnKv9N}MZ`J|A%z+x9c+Hx1`2%m zUlZ;Tr&3M!IDetmdDf(SI?fv&dj$+0{t5ZOYKCtYv89HHq+t}3$SyEJz#qs#MerHM zuP^;KfPWx=9HnMFPdsb3ucFzEkv)<1pLQz6bA}+ z){jP{8v3AOr*UT8k04gWauW)d{vI$dSwDSdCOMNIPHk(0RrdE>VL4gdJJL1?vS+A} z(%~K<8eH?(*ayCj*4*nNZBw@%)3Xa)fBZ&geDA+Qf|LHv>ISGyqBp!blA8?lUm%9(5X_MyEuX$F4UPrtZHLSjcAT|vDa6>oMjUA1M#LI>p0y zEv;hq;{};%YYNStSD|54)r)-L(Q3wmplj~^$QVd-Z?yWSb%B_F3*rSuMDM2ZS>~6> zI0h$UdagfW$QDFN1fj_fPhyjB6tRT%^Le!Npoc!N4yx)|3Rj0;eidX?#M;R4uR^Zdsz zWo(VS3d$7oS+hX@gN|``wOiJu|4q&RH`Dj56wnHXUjN+`_JB_*C0j%={FX4rx6qVj z?;L^L^nF-xBhhX2?NZ)2W@RC+vyh`cGL$QJcNI>Djv_WE>C;d@@i4?%^G%iQ((0FM ze&6`ZAW_kUbM*qn-)Q}aVn^I|)S$lYKnX{Hj2lEtQ{NUQB~p6TGrnnY|e?Vtn%%XwvkwI}F2C)bbFs8=Lm`5z%IYFIKVHXsuT5 z>%_*RwKFeobvsC@nQ_VaWO@tR*4-W_nzzG_JE{nqK6BjgR#TXjKE;CYO#k@(Q-LHXRm_(1(;NP(EU<7iM!rOD9X2}st zm!{nxN_zc%5^Y+riG;kXIE+Z;i!s(%Sfq^l-vssLZ@>5drM|3Dok|xVOX&B-kaS}? zy(JN}$t?Ewdss}#O?7H{I=zN(ZhS@hYp7?FTb6#UaF!Pl#GcRSSy&no@V8p&j0G)m zI$DbUg^f1a3o^sHDodUhpROr9zS=~gAFA0}V_PJ?7f5>llzSj{_iOVlM?`$OlYpBe z>zVK7>RH=C&h2^-5DrW+lX|k!`K{j*(lz9n(R_>Ee9$pfizo@3C?vJ}fYlVS4W~I5 zKtS){FS8~=pM>Rszlj})k-u3;NO$rh4*Sp2-=q;9K7et%v-tgm9CO-cs-Fg1>?!ploSR0EDykff+b(8Q`6^)Q_Tl0# z5TQ)he(8mo`e6gdLI{u$=JGXug*U5m3y6HC78-yC|X`(f>=V1{(`e8_E@Ko%7I`= z!H6;X#p)y9P;->`iuw*c(jU`MDZ(bd>lbDA%2v^S&JC#cPmyNSi$EN%ReQu{J447=ywb0bW9_jL?H}~`bH=KeKgpnd!jSGW*Y?P zjLVsmI%)nT+5)_Wy@sKq;fQgBqQN@GGq(6C_omX|=(imaudX+@rE2@3aSs~Vl7Dm0 zyA58nvqmm&uL)eepG@rqdQqwv;KVP29}1Z!w#}+_t1TL*Gq>?B?t7C2E@~-@Etkc0 zP)%OSx}THBV9*G$rX14a=WBBTXy0ob)c`Pj98kObtA(s>Df4*9X^05IA;Yj*_z6yL ztF5UEh2un^bbueT#62T>S|VMAIb68(XwohY@Hn!)C)`>y?mdu`ej`C3@X8^x_d~A( zo2w8vgW90Lud91EJ!~iJmyvy@K<^tpnn?(u?`3*PLi6B)p&P#OUPqZjZBKh^4&?zD zskyNcA06u3bxv1##RthPo+K&)KVL^kw`K|e1&)gNe1>5@`8qr??E)9eF*0FKJ`cJ2 z$(oB{w(NeZlF)zxPnDts6DZK}<-iY8FDW-wqwi&MGL(1P=aa0bGcy(WB&=hSrd29^ z#oaZPx@I~gK2i056@!w$E3ef_@hLj-^%s=ug57;Z$Q1R?C2-Ou=WgdLez^j7i}I(A zo$BOK)4e?v&`p9Aj#kQl!$;FnKxK-G!Z}A0bk6S_32@#o@X~x=)eO%xf9dRqMIb+5 zRwQvE!p37d$PhlprIHqgLFTw=V+tE&wJWz;4y#CY={`#Iv{N+f^d4#%`H2*ih*?4) z1NAL9r{hN4z*HoL0SkoUobVnKHUPziT-A&KdO!?60<9u_v z+}i|y?Rp?VGRe?jIBht2a{Wm{SJov6S^Cwrhmt2U{3(? zUU45UL=+w}a;>^~D=1G+nQ|O+C$77&WgCr(x_SSr44?hE?HO3vX@IC2l;$06^{sjf zQ-!A%2>EG(1+C^H$?)7h?l1MSw7v?(z|@Z@f#eNd2sSC67UoJT<(+9~rnJluez7v# zsKM!^+(Vb%wIQv+kT7V|t-GJ^a(;;{O09h+<6}^isU<-o@jtkZJ9Br@#NRT+sHPwIV$!X|nCFH$ zo6+<_kf7L05=t$G%t4bajVhXe^P1ax#itD2-h>dAZYnd}--B98Uc06J@jXKSIlaK; zK)4KQ?%iv6Fw8{;4UrhLEzm)M+%lw$ySef30IXnn_^bL$ry4SrcU>3bD4N8zYxD!0 zHXMGJmoX%p7&1k4N}i}>si;dA4NMveZ4tw;cnc$)#oSx3^XT>Mvm{CZ*l^KhWd zRV?3%a!7mYU;*Ee)q{jm3TCZ|*N(U6VjtY9FI@OAUQ?vf7@VmtaguwNE2Zog%vNGW znGLC&^et*DYw zSsuBU>2$b|PH^hTp<~(-` z@KgdP(1S&jv1sNBFaPw@MjikOP|<9=uaftm zg*hPWnkwoKPiJg|L+Z6v;B3N25R z^rsaV5C2e=FD}twA_+V6n3*U6kF9N4-egb9ayUFL#TfKelt8+wYH=a+3grKN{eiw^ zBc?lwBtuZCaS171r1L1>Kg{ZLoaor>Rk}I<@A+>)Xf{*hNiJMOFlZ~KssBoZyRaaM5yJs@KOZmOg#J#bTHp(vcRaX z`1XP0;53jl$T&5=l(#O6@J$y)9gYp5zW&NS0|XTGmwc$PS~!cALK`+sMJY{VqgKqQ zD55zR;j@*+2z#9?mFadgL8NwL63WOq@iisajlPl`x&24H**TlLML|!&@!RR`KSGKWNix|d;dixTBcJl=cC#eIEQtwS!#?^xYnJ@ zlY#va_>a$G!QbS;u{*E2ChaF-4|dcFQ`gDnNL&yFrPYXp{a_=wKbNh^ZsEjR^OctN=w)G8kd2S zef%3fJ0isWQn+kp2@^6Y-nOAyvIlfYW&EMVjpt0Y{At~MvNDsw*D)ok(k57s1&!Z& zNN;cnEbnKDWR-PFC^4a^@Hc925@>YHVF_VGx zP|`;YMtfxcu$AW^ETC8nlg-zK&(~?gyIzpTbehCLrba36SdVJvqY_nCbwQl7MxmUI z?E{}NaQ^=~_|#oOiuZ}%dJQ(mfUn{IbMtIZY=7g91C`BiA*)}ZieZVD<;03o>iw1U zvs_I;TVD>=vJ2XwbDT?a`4O2)_B>u-$XIZ_rsJec)R0DJl;!(!t|C2VC?^&kW0z%s z6ixJnopZVg_X{&@yZczQ_er^z3Qx!&#?WcBoNmMemkRy~V8%_ulLig2LZ$ymeGJ5m zndt%15cGgTU&zG916p|_cz5|!3K>8uwXWaj+lU44vgCAf3A#L}cCO$YA3LrA$%VcnSR-GE7A8R;wbozU)}@Z_El!S{VO{oZIA~f+qv}^B((w)KzzQvp+ zxtCQ0;n2npU}J#k%6Ntc5Vu0Va9$Y|J)y8#oD zbZBxu`ni-$p}F2CyY7rs-`m`WW-oR_erL!3G8CDnUV^&r*k*w6{vso%2Wf z)>Y@f#c=I6RTs+o)xq3ZR+w9Nw89Awx+?i<_q=3#-!|WBc|Tj#Pvx!LDH^r(nZn;O zQ4b|_NvU}*&q#v-$A(c~Dh>mAwVqGaf)`Da$NXNoC*EWyNJO*0W#L}%I&K;u1E<7; zEV?#tFrO+%p*|JjjJ4w&e-QbOSkux9YP}u{wN};oG8*j8$!2ItHX2}$tQA1Yyk}*r z$-;WKYN=wAqCqtiEVh;JkJ7{N^8D)@iTReaJEJ=aoQ5(adOu-G#2?4>{+bAddmo61 zOb{S)k877tm>z%h79b&VU-RbuIq+vFobty-F!H2h5M2B#nMDvB1&_sR`V1tM8V`_x z;XDZLPH_(BdLOGv7b+L1OfG*TPy&kagWcRO{srb))KPxsQ6M|`8FBr)H;=pPa$A}1BhPr5i2tdx2 zAw^~ng71$IKi?Vyh}GvO#jefBJOfC_68nkix@H={@mALYBPuyq3tnBxKH@jwg#CLA zG5Lj=dYDekz}*q?ejD)+QDOaf+X`F23<)Zl*Kn^T_i4JH=p)uSY+jF9J#U{q z2A0Z46QeJ6BsmfMa#>qo22gSW)3*HcS%W}t_>_f$-m~W6Sl&PXCIm`$Y0$K%0j5p+ zR^iN|QdB#>b*{c4x6J%TaQKrLF!o}Bx~=G{P>*&%;Ju|RBaRMtYpwc4CNEh$AZ1Y2 zuL$Xip6c>W>bHD&@)#=Da*D#pw-n_$!Z$3_?SWCwOx&|?%jD(r@$`!>rUO9DrMPt{ z2rNZMcsE>gDaAh-Ch3KL1Z!OJV5mRQ;K(vWibxUhpl|gdP%Xxn=ljYemlT1>id6G? zB$rmhxdIK5aUn1Yh|}gFT)6c*41g(yOv3OH_W6Os)!N_SIUL6cUIVG@$hSmLziwwH zEhHSlF7)Rt?i@wM7Q!Yff+}h1rZ6V+p>yJMonLE#x&h~iSn7ZFHmout*h9^AqD|KW z;t)U*2~r)VOVMkV`oa7q6e_Q&S^J)UB!GBBmo@ndWfF0=3=IDPaGg61mw|uUy8%_P zb6A55OUJQ%LIu8r@|F}LvF72Pd3Vx&Fh*TYLz=wf{Vkg?fldaqTj{itG)Ke?3jKgM_E z6y&HBq~eM~QQUk_+N; zfcrxEX!v$s|0{R&d215AuM#=G%2u+Iw;>KglLiID(ct(GOgiY7)9*zRV-S`!oXY^B zXh3NQ?qyA9?~{QMJYL%E9G}^ zuJ%V8q(ER=Nw;qCfm@-f4k>nYS5w<~X>P!Yr7q4rE@L35J0=aJuBc?V`y9L#%_v%6 z!FF1SprX5R@!%|grfvND!*5Kx8k~L%pyogrzQBSSfys!s9kCcK z_c51tpK99x9lT&5zS7WCVrKdqhc_(^;*w*0c85_k z&ySzww+`VsE>Tz^O=~OFWeMrTvpsDY(~E>47!emaOF=hVE143Jq^uM;GBwWfxBg0P z+(B>96Xo&ECU`z*p>TzeLu(YJ=7snB!}Q$)?TAjig(^{woSKix zN=fglLezc`y2ySs(teKvw$iQ|5kIqvueI)fEtb^XCb~B?1}5$MlS)x!fGB-DABf_5 zXvBPaa|?iY_ZrL(47+h?xD*+PNF@~SfmBiZh8X(&(nDUHEkW@47mTPaA!MSxI!N_;ONDn}*Q6lk;8C=W;9sM~y(;o}@ghgNZ zKOlIvDty?Egy=orUfBP+yR&J461FTH=gE!zc(aI%Wh5;g*cHQo$Bp^s;;&`x@PqYq zcYg-BHnv{D>^fd?5Dya1x-67jrId=}QiYD4fE9X=T;s(=8vG(mdv_rC?Bt%yP*mo6 z%tg!bl3n)y-A^h9#s+$}l9@C}?sG6K$P5h;FyQ*o=TJSnzeENG020%Ge=7sSdyY7# z!7^}n;+e=@`f0GAB=$(>SJq>^lz7G{>5&xe5u=eYGz%Ju@s!@sh_DSjDM)SV-Ny#uEL+hVxtQ8S1+|tNHZOAPx{Rf&P(#y#uA0KLI z^{0RY%e%M-w?ERwn4pxEKYQkw>mh|dB@abAP>J-$Du6hGAPbSj(K!d|RHUtD8N9>3 zxb-HzB8?M$}liM-n@|kge^E~1QyI_ z{hL6{z8g0Ul7ZtsbhKZ7wLDh)a<2X>%F5gY9iNL)B`2%cG$JCirM5}t?5C-99b+2m zLom1cC4C7dGcqcbj@b+OwwSv5(?R5-qFDAgiNzc8AL9b51eUI$IDL`=h}aahAb{e# zV>h1nWu_MjDADYm;J;_^U$!}otl7t1iVh#(oOrgf{8oCgp1tvBB;}WRgefrKYvQ7_6VXth7g`mL1HrjUU}emaNk9dmp@g5BFy zf6!L$*%M7qNj8C8Otnzf296dQXo&>v#zBzl1HgP-Xnr`$rHP0^PMb&>x(P7%b zukH(_XO`r@>$U*TlhvXdD`13b~2n5kCcg!iOl* zqros?H6ah1cJkO4DwFtx4|ej>Dd*ad-90Ln0X1sP`zsZ~-y9YvfyjBRr%4ywUVk*W zxJX{>1mK~TY3l#=M1u+x^(zI@C`|#&DG#<)*RQn(ARXczmDSag3g0QsO0r$+C2w|< zn%6y7m*+^;YX_>LyShH?*UY~Ou7w{Nl5Nx7pf;euWPiY(k1_BsSjR_IaM`z2+0&W|{xYIpx>ZOCwvG^OABh?_6*?_k)~J3b$SxPq_gPlya9#ynZ2% zf>)WAj;^>cdz?hd5Zt(tmt4fy{h%;t1>VAmwbYgr?VdzXsPO@lg8A*nY4HnACw0)oFeOV- zl;#(gHl_R7N?KaA3?9oqE^866vfyAGYITWdoXF`&*6Z9veIvy;NycHHbc;(8c;Cz{ zYF45rfm;XbBLGMMQK$^XMBfK4W{?tzKQ+eZZT0K_IV4YkClF17g=C0Bgeb;YI`rzs zvZZghja%`43nw0eI@k4lOxutm+9(#Q(0^bp81ttrYk(&9o%SbWCR4kno=_V%AL5Sz zwd(R#Tk$m&MSP%oL(|un0dWP}M>qAxIW3uFwez3nP@djxsM{&sGUEKH2FF5Ti~`0E z&pxYs_^j}7Wl-3o-h0|%&Z;=_=<(?1r)B+k%a$bZCs8P8)>RM&*&5Ro@GS%0JM=PC z{w0bK#e@LeF(h&*2Cq|Sc@~!b*`ie(vM}r7iGbbn-uS_1x%EPEZve{v7^D6HaV_L-3wfRDd%HOL?4-Uy3NfooN7?oCwCu5a<{zwRK zw7yuv(31vOpO?qJuuXT$O#oa4;8oZiK&2|oa`{n`IN);)|^imvJ$`7qxqc(%IHT=1UW1WR~oVK1+xgArK8q-CDvl&Dw3Xzf@ zYkfPPDj+vCeOkxMImtpnfKaySY!uiF@eX`bA%Os%$Aw_b=Nf@-mt@TSykz^%%jBws zV6MijfNH_Kl%ja{txvD}&ACEz6U_X!VNDe>fWHvAb{zD(N_<;LX zzvFH0aGMW`Xa?)^X|1^ERV7*1Ro(5~s^?8=J*?Yt*i20q!NFJpqEtDd^JN%o7+OAm zHg3#_l-*T=Mtzrqv9N>ybg#l;-RLQTBcQqvw}^TPsR7lps4b;$n`Ot{udGG>6|fJ+ z$RUB-H_XtB${)q5Js%vBxTUwzDU0SCuV?>Vt>nwT9mdohNV;zHQO2^fprbENLhfGo zqqLAL8yNNTdpBSR@c*Ce0)XvS*w1sZV*%hQ^*B|(V=nG~k>+bO0f5crgSOR)Be}op z_<7Vf=FIK|_()jqHyaRabOa&C_FL|{JP*0?A|Pys0a z{~SU6FoyGUs=n&?QZln!8_#)`OWO7n12%GX9*67NZ-00q+_vSuP)eHXU~r`3Q%~~q zZH$rZ8&jG17zc`_+^qTNgVbMAw|PE7y>>H+-4S$VxQC4}#8w6l)diFYTYBx<(0D1e%+6~-AsHaV4c~St7z@YG_mAYEJ|U-UVo^&RFfSk zK5$pHo12?IxL&po1)Do~k`Uw8EBfPGh%Ju>U?I4a;=uSHQ^hfZwo-GrsVz8XxMZ#! zlYJ21E?fS#z&DLH)Tl~0@>ba4P4sC47}K~&u+H((N_DTeYZnG4^pIT|yTlD7MbMq` zS)9Ko9e4dv62Wr8G>@0z-*lvTrPeb9SO@4<|5ubTT%RnSc%{9Q+%$rI!QU&Wm-j0V zL2(XiIJE%!7^XAu-f3)#dIR^ajz@HGJFy{>Ykf&m2{ou|ANb*AO(6KDJKOc`O`qR& zDh1UMBelw-*N$7)G~&2jVL@OWkByEH^h9po=gK$MSX4^U%)c~vvTI6mo_xqNGgXRZ z(K=GVmm*uDx~Mg29NeYonVX#U8q8yKJ{Jlm7ZTdN{)}b8r{y(5Nrra;!jePCe!>qs ze(>;c-m5*HH)UUHsHTyQ_hYp0{r4s&v2Ud98dBZHctl<1wRHude$QGBJ5oP1P~zbP zwd{i6g)Z%9<}0Op@uRg8da(nUhYQee!S``*Cub-9;;b6UmM6x80wCoZ+!P%I>p$!_ zB4NvY1#WN+Yp#%3$Yy|UM#UZ>Vw~jpM90C$1wlcxt4eer%cne>g4y7fUtlB_BT3+O z;>xg}Tjf7jiy~!J}ZA$42U2BLucV4!l?3W2lOjj@lhe zC2*R$3}TP`b~`3^b)|0hN;%=#Y2)@~1*3Y!YG8O$V$C9+Z_X2)!Y37EW%sz*4R9S* zV%IKqLYk-A)!Tk(TqlL(ieu&$?K8)NSq9Drezi*p5NepqENUYc-i~}D`tpN0ebG#- zacu~m_%tC$ZDTX5gz~w4i1BL!pIeKFxV2BzUYVnzg`QvKnjSUaz|$ zk>QfYyFb#?-nx2|NWP(-d-4|iyIo2t7%|tK<>b_&;*zT&1U5T0-#a?oKc%yifJ5>b zYJvQeQ>|k96_!#nF_(R9&tcD+>ZzKK=q@JY$VH2-#qQ5=rQc|gaE^!wVx?j7F+!`etrz|v_}JSe zLv{8ZM z-ud=D>F_}9;Xf;EL_MS0id^B6)Z<>cmED914659640apsV0heH_Te0w=LO2S0tRDN zuge;gnTn`_V_OPf%(z7Z=rNV@bHAHgXH@3YFD5}Q)LAKP2MnO&r9Tn>D)*gu>%Zz2 zh?t%}wI@zEKik?4HO5^!oj-lQK7uG&2Ws_)!1@BeKHb_);V!yqze=xR5~q!O$}MqE zt8y#)u)ZZ>cQvL4r#(zy0}yytzSymEZ%t6Y*_+fqSdIVkF~Qg1PbvsW5*)V0l2^vFsIZ<**lut+Tew0ERdhU)$z zX844$iWItyybOZE8MZnc@QIf7!d%WZkezG`PJ!j?2g+J{XT+ZPI90=5H?cKqmrQ(Oe>T-*X;ZB5N~PJD4H7G3VlaSe*gQ=b3o#dwbX<7zbgBWpwnfz z#+3N{@pklHgWQA1Qj93t;gpzD|8PdPvuF)1%Xdj`j#+C=#2aaQ#U0*Ft{y8Eg@n}#OM18(nE#&&5;M%qoWfaNX@FpWm?uEFQ;Mi z*#Vl3D!i{}5-NvedWNLdh&;&%>^cM)4@1rKP=#&QM!5_A_yshgUUd-+PY32Qqn2FJ zm@Yd(Tkt5KE4EFwtIl&}YTXYBN{$KN{q6L4w2I)920 zLJym)ewEhZGP+NY1ZA26Ya?p_j}_c=5vi7s-U6d0nG-0ZhylR zvHy@_$@a%ohj)jvdws$bu-WO_M}JI8TI_ntZ_ul&_)E2K^X=oIFs-}^g7s`!zdpgm zCqWSBH6`np1@Z)%r8VV3oZ#z=wH&-1$iBRVC)LLUqDl@(SQ)VeZsXCSXSsAXW1mm{ zm?>Vm_u;kZT39~3U;AX#7mRkFQlh%ITwCh~j!Ouw9AYtAYbm(in>M0!`-#*b`0I)J21SWRN;Q8toR@*KuksBq}LEGu~twlz~Z3>a;+tDSd( zgdvHku4V4xoe4Nta*ydirVq)){rG$ZH1vWATPHq00O-0_eAC_gMOLPnCM`tJRia^b zqerV?b@jv{%;%=39~$=#L9>jW`_^EIEuwgq-1x%h(MebP-2|qp#}+zz((+e@5x=;m zhYubb@X$p!Ss{{0pBp_f-Yc6dENL?zPBOV=675QZ`Xs!OITcKnxOUJb!lVfMEf3O$ z)&ndv+X{{48W@5IyV26W(S_VFU4;DvlJ={7x1(!)yT49xPq4ky`$r?Do*c? zOh4WcViSF_vAg+IJgNd}FFCU_f&x*#icHUWWu8Ga^<>wde~*psUN`O@(D)sLO=VUz zaMM`j)QZh)Fe;h#8>5i*5uCqq-}o=$Bx=WZnuLt;@!E&JVVNzdWXDY^h>I|#mtFje z^bx7YwXgrCuk4cDzL{ALgL_Y<2oCvN7wR5lV3Jm886*u3Ct z;Dv){Bf&Y1_Y0H6OGe$mAs{0caUj^pnQjeO%p|%aU=_SM% zUH{2~FXgq<#h0b>{+3fs282hq71!Ll{G~jTZc(rT*XJ@TtQy&sed?0U9qlm|HwPPX ze6(?#BDFI;o|fH7rVXtqk>DD}Z|Rx7%9j?t6K#U>9Ta9hv@E80eqDiP2hM*q1Ya(g zp=-7nT~4mIGNdY9J_$Lr3eahU3BRmM{S^TJ`j)JgVss{#H%r41v(HYy=|`)=HMQ16 zKjcFsp`GOl@e4hMO6r5(iju)A%y9mMsPobR@5#%#qqoV=Y!co)-&K;dnlrWOTB%uJ zU}&kpGs{%oH|)9~LxuHbWmylxex<)>2LRQrrT_{U==y(jd3Sgp@y+W67N9*@YI_X5 z2m7`{*+a_n$t@8X|8RDzoL6OhK-WH|E!%d&a2AX6Ort*6tjys!bX(}&pwoRKsNDnS z->Sn~Do|S5IQjnkm`$PEOrOWIpT7G_NR|;8&y?Y$=#E-R9@R}g#cZ;VepYq-!VBSM>^;!4KEZI+% z>$z)@E6{z2{leIzs;a9{!VhY{JSlK+5bnvUn9CgU_<%6cIyci4EV&v(6I%e z0=5G`$S%*4eIUXDbu&M=hPpz3?uZ`z2{7Yc35_ule7A6Ui>^c?Mu+#4w$ftD1G^(# zn~(=JON(gpBlqd4sS3-?kEd^2w_Fmx3&zYNr(v^yQD4t6PKNMtrf!suBUcqwyT}^H9MTiGj-)_``aC)RmybscqXci zMEub1wiXcVTI=$BeE1Gqp?SIcw%a_}&KB2e<*eqwv$kB6zFkF_CQ3AEsp1cR{l0ZB z$Pg`%k4!jSDh38q>r%A8WPcwD2a<8fWBdcC;o1ip<(05wtJ1w93$kOgM_zrWr z#6|KEaf~qJzHAKE^m+Hj;=KwFf&B`Pg)gkfYcHa3gHah|QWp3+H@C@)M~a60fx0B_ z-g>&zi;kYg4a!DOK=~cra+6*#PtWy5Ocq&pxae)? zkYtDUsYS7E26>4~Ifv!c2DpAMQhe94IBPi+kd~xaeVl-T=&--NWzqr_Ro{p=ZrXrr z*YcTgv6EChYCpa_ULK}UlbAJ6bCt0(U)jZ`rY{?7dkwxuPHS7)iaYFM;eM^=05$jf zfov%oGPnFf7y)rzkf15OS8I#Q?#cV|m=1eq7S%c@i`5@1qHC|U=4xSoFSCmd%z9^;EZIA7u5n7%!oQ{%R1GXmCDx>!uIE{1`kj&k7mApXWO4EEG>)jI>!9fzaPWfVh(P+Ocp9RR$#U3$Sf?ipD6hLscI|q zHNPNZ``ld3%feS_!A4uY@=-O3V2@r=Xn)?!$FnTDy0^@wuMB)WT0Sr82#!?EgcI^t z5KrrZ$4-X;@9mb(lpP%=qrqy}Azkq%Q}4icHhH6yzz3h4&`DZ(M?TwcQ+WBxd|7Uvow>5$Ag1 z-ln6kn{Ntz)}Hzx+&ouzbY>F#`^IXB+b|tuzn)SBaB1_Dy|Vzdrn60bJ$N$vww;pW z_NciwRS-fzlC2-U=}5}+u5IQR3~KBuxxFN=anLA?pcCtE@wn!w*?=$QX)43&51{*< z?_poZtZmPKY1EvlNzRzZa?VPN^KM4MkNnGj4RRV{|EVoOhC(FpaToF^#na(`UwH2R zwy@p*@w8?d2Ax{0xz;t!sXL2;R(PUrwCIGL!u6fIx-N%mVU&fE1{J!{TS@&6gh+|- z;_iX&On%u|zy$=o@)thiXi`iTa=v)8w=`>Ik!rHn@#Ewq6=c3=*rmvK(>lptyW}nQ zv9Ag)Su;20bA#L~moCQEh0-3>>kN0i6`A~*ywyH3Rb1h+85IW0!iC2roD!S|k}N91F3JiDpB z5r98>=hAUc$ZK7x@#yI9EN+dr7LCZ9-C4w5Np17r1*(u7{;@|S?caGh@e=&*L7(`I zKId<_Fek8kuv_+hd-&>v!jyjH@ehufU?HdxAxWRU|GIwNb>6eoLC1$APLOhZlb#ur zqH3k&QA58%1noCU=6sX>fuR|NaTaR5i|W07y6GifVfDHFXnB$DdwZQ2yRij($;`cqw~sdzZ8CxaJOxPa^2 zmc!m4B~?1%k7;ANL1W0p%A+TC<=6eq(Aqa*X{Hq2QR5h8Rhw1()|IitDORwXNhsVr z+1`Aq{?JC0)_bFdg;s^6n})BnNr+1KYw)aAix=6)Urp*ES3^D4F>@Y8p;q)HDZeW> zf#AhTPlm6$_Gy(n?<2e2`4xAT4eVfJtyv|0zUV9l%@p9-f4Ljcm(%eySYOMBp>YA0 z6c_bAZ_rqI0G*kG#zGR_h}CT~B?z0+1;si4h6lBfsDEtI+Zz?;c@fGka;`5+6ucKs z?r}~?CGEfi?!xmGMBL<7zWTvHJc<>N9!zoL9*plT@_GA`tn&m{7^~i7Z6}TkdpEuv z=tUU*d;_02J|+0{jn2mOxK)(C=!unoc}`&=W+ADJk*j>tupJtcw4xMxCSe1Cyoc@X zj&?vZ(SZaJ{1OiyQXW-K8(b!XmkNr-_J!NJhI#jbMvM-|od(7sk zf@V>XXOfqTduJ{iLBbCD{6vh}L1Q!8_F<3srud#h+gBPIy>H&T^kQ#WQuZuBQH7~) zeLAAJAuB%Se-~^wX}H!lvsP#^7ZpB4o!Ewyn~@QTAWCGBdLpiZc3btg{IHA7_9Tm} z@2$d!6MBkOiCXChdKtOejpMgEhfO;#7t6&hFK>>Jooc96A~-#-mL<`w?OH^1HC=EF z)j?Ty{t7bX;CS@lfH%<_f1wMb(pMexMjdg4qq+;6uGh|+Gd{QsX2Y=KCI$dzjg-;j zHJwqQO$d~)3NF|9jPykpC~8`-=GN2v*JBu?{5aVwm}9VD;u1UL3(U*tMN+9SnWyaA z>C5B@u4B+9EKhC(egp(2O07MT=wY`0)>0lvHrM zm;fPVK9gq^gXa~7sORB*zBjZBBJ+pdS4NVDo#T+Hv&wyGcoSTvYQYmi)^`Gj)Dh}8dlThu{WowWni3a-+)iVntHiUC@{C8sNN%*k^g1m z=-c~z%f<&<%vO&x3?gavew_93)(p#rz|RCrM&1f}?|yGc3PHyEk#tuwUfmL+`<=?( z)~0WPC)7PY0qZ#c>gu1QndZJM^hOIs1bBwTw(av|T7#lNx0+}Mp>tdUnPNN#J5?pl zpm$1Sxq{HR|5RIUYI=;DDjRAFgv~TV(JyBzLhOUvcH66L75Uo5@3q8WxnEUEMUZ`s zH~1Njr&5e?6ZEF7)+1c<)`5l7>{HAsvUS#JuDH_ioOz&E@YE9vCT0LL8Kw&W4eI+yA9uJ=6FnewkUnn1J+uHzIX3Whx^t{4MpYdl zjx9vb5MgExH&;Eb{*J3pXi^&tO=$w;_0Z?}u-U3u^^RE%zB*(+rmpkcXYInbFV~_l zO&m%3$XjTEA*tIZ^)Uah>Zm zU#49p>qxHR2OdEW8ZZ|lK=O}$SPbe^B6q`Q_axmyS8PA_N&n5{d&nFJ> zFP6F*KAlaMs5+o5;+_RQFgle6zM1O|l84mPDh_3(-4CsHb_5U>srKfGNz3~W4kK2f z6YAR-U#Y`UdC-4-=RZ029hMGQHppA#Q3)T{=X}7sOGqAQ`@O9M@{d1n6w@VDWIwfQ zgk~9McRtrpgG)BmV}}~7yvsAjCup&n$l&E`=GnDy^5&KW+}zAj6=Q)hF(dEeUV+GS zKC7-qVzU}CO@LpJYNj~O^Um)9OaMi7YOS`rxLp#NHLfo@OFh*UWpId_+v-Fa)05F{4EYu)r z)%3gCrlff)QXC=ZiCx=q{MaCVE9lp`b$dF%zZM^PlghAScCu2dkjz7wB(LjC43D*o zG+F5TFiDcb51y0uk95fEf&4L5sg~zsMza;*KyzX-t}fAG;I(1~_tp&YcWtAhTEz z!cW78NmZi?Gv<{%L!wsgF6o?}>nAfzfp!Wwx<<6M-_&*vPLf1~Ti9=G8O+m%{g*-7ZNBR1Y7qUrmFt`2yB~m54 z$oub&M6exbG|k||8z9oXspKWqA?TCJ^|zeRO!GR$%QwiBxYx%vC~J!D*>D=+egCte zHxa+ezq19yA<(*o8n@d7l@31UZei7TXm^gXk9}q7LNOWD`<<{EXo>0ampYr}gyg6L zU${N$*t00%SBN8yg{>j)2#v*Hne%Xxk-(5u@M)y5_&emGteIvY=yvI=upXn^wvpue zSDrZ%=aH7%8RZ_Py=hD~pMf8vmq;e`#E>+aym-zHeOps)ujl-Hm1^zdj}`5X=(TySS|0ER*4En@skS6JdmtNJ(Ou^mKsf93J(O&x>@} zxWLO^<6r+$S)Lw37R}Hn7rfuYeD`D}^V{E^a@`Kt*>M|bO-= zYCtnQ7AYo#R8K0JWzae_UZ(Q_Zv05ON&fsm)DRvbNWn#HrRaN@1rM~;M&msGvervQ%P1c%7<~LR-yHdHj8FOF&hgrH0V!WNWT0Q9< zc-ex%BNs64qxk8$`&fJAm1PWH^Nl8hi9|zDxJO*_YHK4zNUzO!b=L=QM&^xrmO- zUrAI3R$z3u7-TFb;=o$(?`^^xk93XG_4n$t_~ZF=RoNufL7z1IY2*m;z7wJAjn^li zJ44dtBg?kZQq2?@+<&CHkS_;M>WWC5q{N^)(c#d^ySeh;Yyp0Q`lkR7yn1Uu!^?GnXvXx#Kene=fobm4QV<_ilYUT00Hf1!+vNgX zOR+Gw=?gy`@AEj5gP(;ZRf+Wn*f;CXkS*>rnE=)_1r;ux5tf z7#a)sK;=+Hq5-7FNZiK6Y&w1AR$+NKz#(Micox^8q{m26EoddXi$ux+05rvcqcBAm z2^Nb(Yp{jDqfmPal6xs z)mSs9f<#|HV72J0p}z+ki5Bo)F12d;%O!QOKCizapc(=2g=Q;+wPww0%H{e>P0%5{ zLV~)%=XwcN-VbxC9&Fo(n{ej4+!L(s6j&tK2R<1t=agEJ{z@9Jj`3cZPt z*f*l>ZlB?2!!W}Tb(uxf$vSBNYAt^?Jf_2ud0#G7-QVvazjHFVV;v#ySC@q|LQA;q z`qGS=v{eJ7{93n~?`vJ@Y;G>McNDs_SxgvJc!P*S^#DQ!ttfQ&zqpG{rb&M>R=`1t zaq^~h&2w@b1}I38M*=rTyec`z_qI8vC4Zf3o+iWytWDBzh!_~MwtKDMR-ML9MTzkB ze?9z2#$hOuUrHw-?q?;klFadqwE$1b$;2s%F>Cq52k_MJp1WDj55cKC)l~Co1Mhgd z1|#uOoW(O|v^kJ+6Ls*aXp?#ok1c2}4j#JrNB6VnGsdM?(fL3+gbpd?mIpQ6qZO$L z^@FDD1#~M?qThez$@Xrkfk~`m{}4TmjlO_a&3G)#5bHooTU>EeVk0)GN zPs86?Ol0*Qc`2TOAgqf-a?{Q8e5J?1a@V2pp4Qo^YC=N7D{W~ZCW{dIFkO?h&b}zJ zT_}YeQx;2JaW&;u>JvZNytX_B;n_&>&rB)kQ6u4@<`JK(?DW;}IHQ@Ei9n%b+w(u8 z)B%h(X&}-K{)q^wM6Vsa+X*fi^Ru^TTERYg*S^`y*mP8TcdNLD&1;Q5r2u&{eCvsK)?xS8;XsE&zt;=$h40hFTb6|NVe+iUpkwd@ zyc>bVhv)VVeZLv6UfsLL@Lc(&?1!bK!!6elbOt)^WdQoyO$ee^=o$}d9xS=+^CWJG~7SLF>=7W;Q7!!cP%EHo^-bYeV) znONm{$qzs|_c?oX=?L1&U$;)s_;=Csi-Ulc|1o_s-(pSBo1cLv zb=QyJhDhRlP|KGHNJU$4>%$Zc{|>#s9T>$?(kG;ngt+(BL6=H2*5VfP+kPK@@T$Jf zp1~BKxm*_*Dziv){}?h>QaLntnYssyv=|E z=~b!D^CfN+|LJUR(@=8h`@s;uKb*}E3x2?v5QVPV&-ri98dK6x^?ax>H&MMm+iiGe zFU&+fvtPu?&_1A!TbIsz66Z1d-jc|O`iwb~>2$Zt4eo5N-sRD(>cxxEvGTClX?F{Q za(=O5(|QKwIHa2Feo4f@+NV!0CHQ&#jabKXDp;9vvTZ?toV##eUkMa{7fg2nz3$^Z z=-e4E#^B`=%$mYpYp;RZS=6FW)M#+sOhTDx38^12V!6bx40D$8gc(&`;o7C{yhxtz ziV(CZsrfm?A^ULE#_+0DyS+?78+1KUS%bBs7(r`B2Ptcz3o?9V*-j(KM@ZH7^*CDk zbcRBkQfP@YoQ^QNv!Q14k$q+Ll)-z8jER*RmZ|;@JH-`53|G3(P@4N8_WaMYtU-^H zS9D=gd!?0;kd|5YWGs1{{)Z*K)D}V&1F4crX>Pkbo@ST4O_kF%o=2iHHNxZvyPYeQ zY!Feqy!stWt}6(fDS?|dmV^n@m4lJ982+(tiOfby}Fcpd0C z{h;53yK`#&OH=3#@KCV;PnI$2IVt`PW&_!1uW$F_cTjpTz z_|{IDLcW*V`6rOngYqb-X#b`$b6wO}vg^sbTziLHy7wjx_h0Ec_dGS+27|xFbo(O# zq$7SM@50OFSgRkt@mxc0dC>3-Rf)77z=o;zwO~!y!9P6&acz73vYHogWei;x92-d< z7WY`=aq7<^OIyV*R7$5wp`2S6z4Q0w2Qr=p>VB|fX36-PcK+8enPmJKF%wS8@1T>M z*Sgkq#sN4P{B~;?1B%P^j~H_by_CRFRZ`S0y{j%WB6UOUCsEfw?AHHq(onmV;MLxD zp zoMgFBUWdYRQ~co&qfHmh6RD-(t6{Hws=d{=vIC`j3GCOzr8lL8{-c;vuiZwuyZN4r zg!37UrQHmumNMNHZqLdqDx@nr8{a3&9s6)lUSpmY>m!dYglhr5;e&_%r1D=k)X-;I z1JIu2e?E{~J?`~<#7^gZbuRWSX3Ur9Un8{>+Rgqb=RbOW$BD}i@2nPd>u7C_7-Ek0 zqSfykVNNgHt8txWVX?=8oW*_| zZJ23Py;BT+=p+w-$+R32nIB4@I)+0oB8w0FZja6m$SsAn;`gi0hAzR{nq%MiSmzbw zSXH}FgXig?LT9e4()YK%NCj#>Jo5Z2DO86IE_qzH+E;k+m5rnpJIqbruDM;9!cF+1 zGD~jRi{g08nh?gBWb%jjHT9P1kK;*-&ZqOHfj15-g!WnrXI3f8$?$h0guV~ETK_ED zCv^B*KX}1*|HKcHyV6%F9M1d0L7qrRBDCzqRJ>?WhV7>DPd(T%>4ioaCqgHG4a8f> zw;MvWHq`!rgbjbsc*pIxR~o~6-LMLGqw*1q4YIV~;A+d--5d-h_Xoq5ti)mZ2FP8e z{?%Z*I5sI0^P~BUG$<{2RW5!Lz-H!&fi~+oM=Vkoqq>lHAyq%XQv9+cna zLk{P$#hc;^?~S~8`Tlo;N$n4MzkQm8P@u`<{xxz&AE%b=TPZ11KHhUs14l76gw7A( ziP_D&dG7zV@Zu#Q5`ChR!4!DRRVH%oYjg<3Ms16&q!7`WA!6I`yGh$8G8L%n)lD!; zyMs)EpjYocmF#DH$JWtE4-pRE7`+xB>8X%PQ=sbpJ5tvnh?D>6+M{``wCc$t@Ah@R zT?;?jPl4qqz0&*NtptZ*W6M+v{8Ij}&OzNvTSo3E*ywf7^H-ry-(IONgYdE_V8(M) zX+O}b!gq$Z*hOq4CZlP&DkD_~iV|>D{3mXdS4H{7;PKt>%vvXq@E3fXR^k@F{h9}I znQ$f;Uc0E|!t(S!C9AQrxrSNVC!?IV2ooX7ypgy^K3I=%D6Vq|IY)WMo?XC|09{lPcDR-fkck{I&wv*cFu89K{4h-=(th3G^%LPy}D`BTP z(3t*GmnOPB!quXikVOd+hz9%&&BX7=DC(`?Kn^ zvY)0Q2Owi$4>M_sE`|wyt)<~J+z1{;_2*Ok9S0&(Ycy zxKM2bLvJEW1XwkoHf;cDTDZ;!8okT;4g;T+pe)x2agn%?I!5RzkISO_)@n_1(3Een z%)ZmgO$M68B&K(_6|}pxjq_gx)&v|3|H(}5vToAvyzIYhquDOg{|U~(XJD`fYm>`5 ze-nwP3^>u*)XaJ1xJ}o-ivA6@`5S$vl9>*PnA-EJCb^Vw7Vd!F?RQ4(GImO`)MqDM z7DnPLWDgBQlj0wDH^TOgERRMCV(9DCCR=#^L5-_jb2aja!L^@y95cfTPQLp^TQd9uf;kX`p+ERv_}iT?>i@t9z!!Cw zzW6`BsJeFe(v2&4Y>Ae8b6>qzBVvfAuyDsiSHeQvTll3_4!4uKt3#GuvW#2b14-L&FL z^o+)vh^eVj6M^RQtW=LaWtV=yJvcQrRb?C${K!<6g-tQMW@-LX7w(bl%G>g&o9dpn zLQESGltg+(_g~`OT^@)(%9s4KhfpopXC3y>y+-_x!nunF0iq=gU*Q)~E(_ha_kXHn zCs6CZY+#juTArd)c`Xw6*@6vUj$2XIHRSX#*IXW#(wpX8MMl^C_g&9(RUh*_q2n^; zFqGa8po)2DE|3A*+v+2cC}E|MY3hxX*~Pm(Bwo5Kw8k*6DQziPlrVZ?w6)kW2|xJp zdmOUxWxAHs`>I3KZr$O2W@6(}rsBa#4`U4=Q0r(xRhQ1O>Rr6v+A1^TAQ+M}YR*=S zsyXsL!mZh}Ynm3b=tryy2_K5$3h*9Y z4O^)PJ>7(kz%tOt-$3>VK1$f~nlzd&`}L+j`ZdoEffha4z3*<6Qop|HwS?%}$5E1k zHSf#&T~YBWWAnUv{7E6wi9sYYrx-e11zK)6jq)g&j)mkGHv^>-#r3n-JCMV!4gq8Jhy*>}_X0)OXC zqChBZm2ZjNvZ<3Usr-6xq^!@WZx=~@EE}O7RG!z9?&66#qr*}f#aLm9qBH-J-xvgV zhf%3O_8Y-^q3nN9>wr5Z`YWL)L{@i$PV|TA!qO+_xWQVAB#!yq`)V^ZY{PM!yz(UgJoJ~}c zM)^m#{e-MCavnN$P$F?WTuo?oMTX>hAQDCx2E`d;PJ!0j06D_wK*Im^o=53CK#&rF z%CSegm(c-g=#jczGQ?8bi!Q-+u(A9IOiUJiIz`74zZTbYJu1oVz0ryo`WEI&O2iTN zOi+(66H|o-7gllJ7eXplz4peQBc|uey8I_+1qQXB6&d%_@8?Pt;C!Qz@BF2J>{ZF! z^Ya9>(bf1SfYN5c7TmH<-ne#&W@A20x9iMAex1fU3#YKJ|LYHrb(`UhrBGOq7LUTR z2iusqTz!u{0Q8Kj63gEJMYI=SI#qiNu`$-s+=bPac>spZ!Z7q*lHao(`#ZnsGWd8Y z8d|`Hz7vDq@vKLNS+ICYj#-)SOB0h?%Tm3e6hh_(a~^Db=?VAvG5tb^K|H7uq=?7(|KTDYAUr98_MiVRcB8F`@cB5`Iag%*rP6`Ye*Cl)g@I2iw3onn^ zqGs@W7;GTLkjvA40bOTjVO?DjEvB9FlkV6uv;1C{8AnA)mpR9p?oT$efur0Zx)4Ld z-@k({VsLw%H2d!D!hV8Z0Y+5c7p))~Qg5M)Ed(io5P&do>6MC7&sa9o(S1uyr?@K& zSBg2W9Wtm+$+T#25g2iDqP~hwY>Pm;WVb`&&973xqVbJv8JjJZaxShXo3fGSIHmX zE7>^JYo2MP8>TA37Z?<`4{{HnsO*ks(kee$-bjT$l3dmh% z>O%+s&>S$dS0v>Khcavk-7=P zu)%^wbfyey6gqU9DfvIi9wsPK&1`wST)O&Vh}7G>iUa?aQ#Tj97BvP`lZ zoNrj?d`}$57|KFsoZKz6=$vv=CkHul?SVummz>ju@2sLu$sC2pY~S}M{ct{i!RPUL zyx#B6`}KOf-p?1$k)+E2Mjk&pX!CMC?)s1pXg#D!CG$w-qZA-1K{ueCT;Xdp)qj7d z?d)WcAg#<`y_v~?;49r?6$gT^w1iHVpABE1y5083lui}p)b{~Rly1F_lJ-Et9SCtV=ZNp9dgAH%S}FL4Q$6@*+Be3Lqjs1;5~!^NuDAR5zk ziw!pkw+)LOe+7Y9IG8-fZJ+2ug_yGUTz3!U7pZKyT6tm*j)*-pF?RE)rmDZNn=;}H z>y10~KjE6M+=1!;>0aa6P}DAeq+cQ8(X~~cr4+F#G{z6-D7Ql>WF6n#1HN&5a*_fn z`nfLh;ITwF-BE+X)j578`e|B;Y^`^x;V)Hh6=&7?)Y)HKFO{2RjNwRr0%E#Lo9F|P zP;zgvqtL0SfZFCuB5bVkilRU(sl>jgnU z7o#1JhUFOei&?W_E$#ygPp;ne%7P75nMvyE7w*nbQ$3XJ%=}BkqCyr{(<{irI=R=O z7P7FM_1fM&xECEX%)ILf##KCaiS1^lK5z8bS}UocejHcPJ{8rukr_q^Er{u1NmKfS z z76S%Gbi0HfEx+*QLG7N?8)b&_g@qHE+&}#BY=$5Sc)1xU($WKKt>0UlX81-L+95jvEG;(DU{~wJQTQ@-;$b_qKRCA-62L zLtBlnYZ4i!{3)W4cT7Ge6x*PHAZ86QbijK{C8$bT6mN`;;cKo!^U0Ds#K@!@nSCT; zyJDKAbV?wl)(xH!-eIC@2LZ_iF+UK;G#yEvSQm$w)4aK!t{1gou&?Gr+cpV7+BZEy+C1lyxQ>6VW1S$6`U!ThUFtFOaE77a+VPlO zm}$ZR#H4|-K4-=lUYlFnguH`9t56Sy7dWT;0Br{S^ z=8`@)1JhPk^P8>ug3r-v-LBT>z%*xT?5X zdqZOv5(#omwa86d?LQ*eZF1lTLWCVzSAC9F#Vz0Sj_k#Y55tCDJC>;@(0F1<{&{7lm;JYn>;&|iZ3A@iA-ev=Rz|8O zJn`hUk$Yw3!;h=B&m|r9l~9QB<&k>*WU%g|2S+tk+6-HH7C-(O45PzLkAk9p?u=Wv zk7XSbjsfzpiH;g;1#MF%iZI*!)A7{6Q2{|d<`fls1P%UvGu3vv*DFlu%lpGk*ClyO za02yKy}ebR(|>eS7@;~66o~IE{UQ|%X-FAIAis$Gi`(dZ7h@%1mrO03-s-hP$nrwF zn$1!sZS)3Q_$-?_oLJ98RC~cQBhm45d+Ws?iq=tH(Dqc0s*FqT9#Hl@0WPH*YSrd% z__~2R%(RD}W75MXVW}#c-A3 MixEI?rbY|?4 Date: Tue, 21 Mar 2023 10:46:43 -0700 Subject: [PATCH 119/421] backport of commit 18821f652e2131681f4f6597783b676cf60c42d2 (#16720) Co-authored-by: Tu Nguyen --- .../config-entries/service-defaults.mdx | 38 +++++++++- .../proxies/envoy-extensions/usage/lua.mdx | 69 +++++++++++++++---- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index f14b0c63e643..e1e15cf7acb2 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -57,6 +57,10 @@ The following outline shows how to format the service splitter configuration ent - [`TransparentProxy`](#transparentproxy): map | no default - [`OutboundListenerPort`](#transparentproxy): integer | `15001` - [`DialedDirectly`](#transparentproxy ): boolean | `false` +- [`EnvoyExtensions`](#envoyextensions): list | no default + - [`Name`](#envoyextensions): string | `""` + - [`Required`](#envoyextensions): string | `""` + - [`Arguments`](#envoyextensions): map | `nil` - [`Destination`](#destination): map | no default - [`Addresses`](#destination): list | no default - [`Port`](#destination): integer | `0` @@ -120,6 +124,10 @@ The following outline shows how to format the service splitter configuration ent - [`transparentProxy`](#transparentproxy): map | no default - [`outboundListenerPort`](#transparentproxy): integer | `15001` - [`dialedDirectly`](#transparentproxy): boolean | `false` + - [`envoyExtensions`](#envoyextensions): list | no default + - [`name`](#envoyextensions): string | `""` + - [`required`](#envoyextensions): string | `""` + - [`arguments`](#envoyextensions): map | `nil` - [`destination`](#destination): map | no default - [`addresses`](#destination): list | no default - [`port`](#destination): integer | `0` @@ -128,7 +136,7 @@ The following outline shows how to format the service splitter configuration ent - [`localRequestTiimeoutMs`](#localrequesttimeoutms): integer | `0` - [`meshGateway`](#meshgateway): map | no default - [`mode`](#meshgateway): string | no default - - [`externalSNI`](#externalsni): string | no defaiult + - [`externalSNI`](#externalsni): string | no default - [`expose`](#expose): map | no default - [`checks`](#expose-checks): boolean | `false` - [`paths`](#expose-paths): list | no default @@ -666,7 +674,7 @@ Map that specifies a set of rules that enable Consul to remove hosts from the up ### `TransparentProxy` -Controls configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. +Controls configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. You can configure the following parameters in the `TransparentProxy` block: @@ -675,6 +683,18 @@ You can configure the following parameters in the `TransparentProxy` block: | `OutboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | | `DialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | +### `EnvoyExtensions` + +List of extensions to modify Envoy proxy configuration. Refer to [Envoy Extensions](/consul/docs/connect/proxies/envoy-extensions) for additional information. + +You can configure the following parameters in the `EnvoyExtensions` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Name` | Name of the extension. | string | `""` | +| `Required` | When Required is true and the extension does not update any Envoy resources, an error is returned. Use this parameter to ensure that extensions required for secure communication are not unintentionally bypassed. | string | `""` | +| `Arguments` | Arguments to pass to the extension executable. | map | `nil` | + ### `Destination[]` Configures the destination for service traffic through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. @@ -1046,6 +1066,20 @@ You can configure the following parameters in the `TransparentProxy` block: | `outboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | | `dialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | +### `spec.envoyExtensions` + +List of extensions to modify Envoy proxy configuration. Refer to [Envoy Extensions](/consul/docs/connect/proxies/envoy-extensions) for additional information. + +#### Values + +You can configure the following parameters in the `EnvoyExtensions` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `name` | Name of the extension. | string | `""` | +| `required` | When Required is true and the extension does not update any Envoy resources, an error is returned. Use this parameter to ensure that extensions required for secure communication are not unintentionally bypassed. | string | `""` | +| `arguments` | Arguments to pass to the extension executable. | map | `nil` | + ### `spec.destination` Map of configurations that specify one or more destinations for service traffic routed through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. diff --git a/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx index a68e7a822518..496b7d5fa58f 100644 --- a/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx +++ b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx @@ -164,22 +164,65 @@ In the following example, the `service-defaults` configure the Lua Envoy extensi ```hcl Kind = "service-defaults" Name = "myservice" -EnvoyExtensions { - Name = "builtin/lua" - - Arguments = { - ProxyType = "connect-proxy" - Listener = "inbound" - Script = < Alternatively, you can apply the same extension configuration to [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries. + +You can also specify multiple Lua filters through the Envoy extensions. They will not override each other. + + + +```hcl +Kind = "service-defaults" +Name = "myservice" +EnvoyExtensions = [ + { + Name = "builtin/lua", + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOF +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-datacenter", m["datacenter1"]) +end + EOF + } + }, + { + Name = "builtin/lua", + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOF +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-datacenter", m["datacenter2"]) +end + EOF + } + } +] +``` + + \ No newline at end of file From 16a19762f43948ac984de0ec36ef3fa6209f3a10 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Mar 2023 15:22:13 -0700 Subject: [PATCH 120/421] Backport of [NET-3029] Migrate build-distros to GHA into release/1.15.x (#16718) * backport of commit bc70de1cf7ef41aeee2161ce079f04ad22f1c209 * Manually fix build.yml bug Signed-off-by: Dan Bond --------- Signed-off-by: Dan Bond Co-authored-by: Dan Bond --- .github/workflows/build-distros.yml | 43 +++++++++++++++++++++++++++ .github/workflows/build.yml | 46 ++++++++++++++--------------- 2 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/build-distros.yml diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml new file mode 100644 index 000000000000..9a43a7f52065 --- /dev/null +++ b/.github/workflows/build-distros.yml @@ -0,0 +1,43 @@ +name: build-distros + +on: + pull_request: + branches: ["main"] + push: + branches: ["main"] + tags: ["*"] + +jobs: + build-386: + strategy: + matrix: + os: ['freebsd', 'linux', 'windows'] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Build + run: GOOS=${{ matrix.os }} GOARCH=386 CGO_ENABLED=0 go build + + build-amd64: + strategy: + matrix: + os: ['darwin', 'freebsd', 'linux', 'solaris', 'windows'] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Build + run: GOOS=${{ matrix.os }} GOARCH=amd64 CGO_ENABLED=0 go build + + build-arm: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0c1dbbd82da..8c49acbdd8d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: pre-version: ${{ steps.set-product-version.outputs.prerelease-product-version }} shared-ldflags: ${{ steps.shared-ldflags.outputs.shared-ldflags }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: set product version id: set-product-version uses: hashicorp/actions-set-product-version@v1 @@ -60,7 +60,7 @@ jobs: filepath: ${{ steps.generate-metadata-file.outputs.filepath }} steps: - name: 'Checkout directory' - uses: actions/checkout@v3 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Generate metadata file id: generate-metadata-file uses: hashicorp/actions-generate-metadata@v1 @@ -68,7 +68,7 @@ jobs: version: ${{ needs.set-product-version.outputs.product-version }} product: ${{ env.PKG_NAME }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@3.1.2 with: name: metadata.json path: ${{ steps.generate-metadata-file.outputs.filepath }} @@ -92,10 +92,10 @@ jobs: name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Setup with node and yarn - uses: actions/setup-node@v3 + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 with: node-version: '14' cache: 'yarn' @@ -157,13 +157,13 @@ jobs: echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@3.1.2 if: ${{ matrix.goos == 'linux' }} with: name: ${{ env.RPM_PACKAGE }} path: out/${{ env.RPM_PACKAGE }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@3.1.2 if: ${{ matrix.goos == 'linux' }} with: name: ${{ env.DEB_PACKAGE }} @@ -181,10 +181,10 @@ jobs: name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Setup with node and yarn - uses: actions/setup-node@v3 + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 with: node-version: '14' cache: 'yarn' @@ -232,7 +232,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 # Strip everything but MAJOR.MINOR from the version string and add a `-dev` suffix # This naming convention will be used ONLY for per-commit dev images @@ -266,7 +266,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: hashicorp/actions-docker-build@v1 with: version: ${{env.version}} @@ -286,7 +286,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 # Strip everything but MAJOR.MINOR from the version string and add a `-dev` suffix # This naming convention will be used ONLY for per-commit dev images @@ -323,15 +323,15 @@ jobs: name: Verify ${{ matrix.arch }} linux binary steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Download ${{ matrix.arch }} zip - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{ env.zip_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v2.1.0 if: ${{ matrix.arch == 'arm' || matrix.arch == 'arm64' }} with: # this should be a comma-separated string as opposed to an array @@ -353,10 +353,10 @@ jobs: name: Verify amd64 darwin binary steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Download amd64 darwin zip - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{ env.zip_name }} @@ -380,7 +380,7 @@ jobs: name: Verify ${{ matrix.arch }} debian package steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Set package version run: | @@ -391,12 +391,12 @@ jobs: echo "pkg_name=consul_${{ env.pkg_version }}-1_${{ matrix.arch }}.deb" >> $GITHUB_ENV - name: Download workflow artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{ env.pkg_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v2.1.0 with: platforms: all @@ -417,7 +417,7 @@ jobs: name: Verify ${{ matrix.arch }} rpm steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - name: Set package version run: | @@ -428,12 +428,12 @@ jobs: echo "pkg_name=consul-${{ env.pkg_version }}-1.${{ matrix.arch }}.rpm" >> $GITHUB_ENV - name: Download workflow artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{ env.pkg_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v2.1.0 with: platforms: all From f6ecffb5bdaa60a290fdd9706181638260b1b3bd Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Mar 2023 16:37:02 -0700 Subject: [PATCH 121/421] backport of commit 17904bac70424ccec7a90993f252acf94832bbf8 (#16725) Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> Co-authored-by: David Yu --- website/content/docs/k8s/helm.mdx | 72 +++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/website/content/docs/k8s/helm.mdx b/website/content/docs/k8s/helm.mdx index 602efba9e6de..69614cd2ee97 100644 --- a/website/content/docs/k8s/helm.mdx +++ b/website/content/docs/k8s/helm.mdx @@ -58,7 +58,7 @@ Use these links to navigate to a particular top-level stanza. the prefix will be `-consul`. - `domain` ((#v-global-domain)) (`string: consul`) - The domain Consul will answer DNS queries for - (Refer to [`-domain`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_domain)) and the domain services synced from + (Refer to [`-domain`](/consul/docs/agent/config/cli-flags#_domain)) and the domain services synced from Consul into Kubernetes will have, e.g. `service-name.service.consul`. - `peering` ((#v-global-peering)) - Configures the Cluster Peering feature. Requires Consul v1.14+ and Consul-K8s v1.0.0+. @@ -119,7 +119,7 @@ Use these links to navigate to a particular top-level stanza. - `secretsBackend` ((#v-global-secretsbackend)) - secretsBackend is used to configure Vault as the secrets backend for the Consul on Kubernetes installation. The Vault cluster needs to have the Kubernetes Auth Method, KV2 and PKI secrets engines enabled and have necessary secrets, policies and roles created prior to installing Consul. - Refer to [Vault as the Secrets Backend](https://developer.hashicorp.com/consul/docs/k8s/deployment-configurations/vault) + Refer to [Vault as the Secrets Backend](/consul/docs/k8s/deployment-configurations/vault) documentation for full instructions. The Vault cluster _must_ not have the Consul cluster installed by this Helm chart as its storage backend @@ -210,7 +210,7 @@ Use these links to navigate to a particular top-level stanza. The provider will be configured to use the Vault Kubernetes auth method and therefore requires the role provided by `global.secretsBackend.vault.consulServerRole` to have permissions to the root and intermediate PKI paths. - Please refer to [Vault ACL policies](https://developer.hashicorp.com/consul/docs/connect/ca/vault#vault-acl-policies) + Please refer to [Vault ACL policies](/consul/docs/connect/ca/vault#vault-acl-policies) documentation for information on how to configure the Vault policies. - `address` ((#v-global-secretsbackend-vault-connectca-address)) (`string: ""`) - The address of the Vault server. @@ -218,13 +218,13 @@ Use these links to navigate to a particular top-level stanza. - `authMethodPath` ((#v-global-secretsbackend-vault-connectca-authmethodpath)) (`string: kubernetes`) - The mount path of the Kubernetes auth method in Vault. - `rootPKIPath` ((#v-global-secretsbackend-vault-connectca-rootpkipath)) (`string: ""`) - The path to a PKI secrets engine for the root certificate. - For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#rootpkipath). + For more details, please refer to [Vault Connect CA configuration](/consul/docs/connect/ca/vault#rootpkipath). - `intermediatePKIPath` ((#v-global-secretsbackend-vault-connectca-intermediatepkipath)) (`string: ""`) - The path to a PKI secrets engine for the generated intermediate certificate. - For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#intermediatepkipath). + For more details, please refer to [Vault Connect CA configuration](/consul/docs/connect/ca/vault#intermediatepkipath). - `additionalConfig` ((#v-global-secretsbackend-vault-connectca-additionalconfig)) (`string: {}`) - Additional Connect CA configuration in JSON format. - Please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#configuration) + Please refer to [Vault Connect CA configuration](/consul/docs/connect/ca/vault#configuration) for all configuration options available for that provider. Example: @@ -258,7 +258,7 @@ Use these links to navigate to a particular top-level stanza. inject webhooks. - `gossipEncryption` ((#v-global-gossipencryption)) - Configures Consul's gossip encryption key. - (Refer to [`-encrypt`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_encrypt)). + (Refer to [`-encrypt`](/consul/docs/agent/config/cli-flags#_encrypt)). By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually. The recommended method is to automatically generate the key. To automatically generate and set a gossip encryption key, set autoGenerate to true. @@ -289,17 +289,17 @@ Use these links to navigate to a particular top-level stanza. - `recursors` ((#v-global-recursors)) (`array: []`) - A list of addresses of upstream DNS servers that are used to recursively resolve DNS queries. These values are given as `-recursor` flags to Consul servers and clients. - Refer to [`-recursor`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_recursor) for more details. + Refer to [`-recursor`](/consul/docs/agent/config/cli-flags#_recursor) for more details. If this is an empty array (the default), then Consul DNS will only resolve queries for the Consul top level domain (by default `.consul`). - - `tls` ((#v-global-tls)) - Enables [TLS](https://developer.hashicorp.com/consul/tutorials/security/tls-encryption-secure) + - `tls` ((#v-global-tls)) - Enables [TLS](/consul/tutorials/security/tls-encryption-secure) across the cluster to verify authenticity of the Consul servers and clients. Requires Consul v1.4.1+. - `enabled` ((#v-global-tls-enabled)) (`boolean: false`) - If true, the Helm chart will enable TLS for Consul servers and clients and all consul-k8s-control-plane components, as well as generate certificate authority (optional) and server and client certificates. - This setting is required for [Cluster Peering](https://developer.hashicorp.com/consul/docs/connect/cluster-peering/k8s). + This setting is required for [Cluster Peering](/consul/docs/connect/cluster-peering/k8s). - `enableAutoEncrypt` ((#v-global-tls-enableautoencrypt)) (`boolean: false`) - If true, turns on the auto-encrypt feature on clients and servers. It also switches consul-k8s-control-plane components to retrieve the CA from the servers @@ -316,7 +316,7 @@ Use these links to navigate to a particular top-level stanza. - `verify` ((#v-global-tls-verify)) (`boolean: true`) - If true, `verify_outgoing`, `verify_server_hostname`, and `verify_incoming` for internal RPC communication will be set to `true` for Consul servers and clients. Set this to false to incrementally roll out TLS on an existing Consul cluster. - Please refer to [TLS on existing clusters](https://developer.hashicorp.com/consul/docs/k8s/operations/tls-on-existing-cluster) + Please refer to [TLS on existing clusters](/consul/docs/k8s/operations/tls-on-existing-cluster) for more details. - `httpsOnly` ((#v-global-tls-httpsonly)) (`boolean: true`) - If true, the Helm chart will configure Consul to disable the HTTP port on @@ -463,7 +463,7 @@ Use these links to navigate to a particular top-level stanza. This address must be reachable from the Consul servers in the primary datacenter. This auth method will be used to provision ACL tokens for Consul components and is different from the one used by the Consul Service Mesh. - Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](/consul/docs/security/acl/auth-methods/kubernetes). You can retrieve this value from your `kubeconfig` by running: @@ -574,7 +574,7 @@ Use these links to navigate to a particular top-level stanza. Consul server agents. - `replicas` ((#v-server-replicas)) (`integer: 1`) - The number of server agents to run. This determines the fault tolerance of - the cluster. Please refer to the [deployment table](https://developer.hashicorp.com/consul/docs/architecture/consensus#deployment-table) + the cluster. Please refer to the [deployment table](/consul/docs/architecture/consensus#deployment-table) for more information. - `bootstrapExpect` ((#v-server-bootstrapexpect)) (`int: null`) - The number of servers that are expected to be running. @@ -613,7 +613,7 @@ Use these links to navigate to a particular top-level stanza. Vault Secrets backend: If you are using Vault as a secrets backend, a Vault Policy must be created which allows `["create", "update"]` capabilities on the PKI issuing endpoint, which is usually of the form `pki/issue/consul-server`. - Complete [this tutorial](https://developer.hashicorp.com/consul/tutorials/vault-secure/vault-pki-consul-secure-tls) + Complete [this tutorial](/consul/tutorials/vault-secure/vault-pki-consul-secure-tls) to learn how to generate a compatible certificate. Note: when using TLS, both the `server.serverCert` and `global.tls.caCert` which points to the CA endpoint of this PKI engine must be provided. @@ -653,15 +653,15 @@ Use these links to navigate to a particular top-level stanza. storage classes, the PersistentVolumeClaims would need to be manually created. A `null` value will use the Kubernetes cluster's default StorageClass. If a default StorageClass does not exist, you will need to create one. - Refer to the [Read/Write Tuning](https://developer.hashicorp.com/consul/docs/install/performance#read-write-tuning) + Refer to the [Read/Write Tuning](/consul/docs/install/performance#read-write-tuning) section of the Server Performance Requirements documentation for considerations around choosing a performant storage class. - ~> **Note:** The [Reference Architecture](https://developer.hashicorp.com/consul/tutorials/production-deploy/reference-architecture#hardware-sizing-for-consul-servers) + ~> **Note:** The [Reference Architecture](/consul/tutorials/production-deploy/reference-architecture#hardware-sizing-for-consul-servers) contains best practices and recommendations for selecting suitable hardware sizes for your Consul servers. - - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable [Connect](https://developer.hashicorp.com/consul/docs/connect). Setting this to true + - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable [Connect](/consul/docs/connect). Setting this to true _will not_ automatically secure pod communication, this setting will only enable usage of the feature. Consul will automatically initialize a new CA and set of certificates. Additional Connect settings can be configured @@ -713,7 +713,7 @@ Use these links to navigate to a particular top-level stanza. control a rolling update of Consul server agents. This value specifies the [partition](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) for performing a rolling update. Please read the linked Kubernetes - and [Upgrade Consul](https://developer.hashicorp.com/consul/docs/k8s/upgrade#upgrading-consul-servers) + and [Upgrade Consul](/consul/docs/k8s/upgrade#upgrading-consul-servers) documentation for more information. - `disruptionBudget` ((#v-server-disruptionbudget)) - This configures the [`PodDisruptionBudget`](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) @@ -729,7 +729,7 @@ Use these links to navigate to a particular top-level stanza. --set 'server.disruptionBudget.maxUnavailable=0'` flag to the helm chart installation command because of a limitation in the Helm templating language. - - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul + - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](/consul/docs/agent/config/config-files) for Consul servers. This will be saved as-is into a ConfigMap that is read by the Consul server agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -906,18 +906,18 @@ Use these links to navigate to a particular top-level stanza. it could be used to configure custom consul parameters. - `snapshotAgent` ((#v-server-snapshotagent)) - Values for setting up and running - [snapshot agents](https://developer.hashicorp.com/consul/commands/snapshot/agent) + [snapshot agents](/consul/commands/snapshot/agent) within the Consul clusters. They run as a sidecar with Consul servers. - `enabled` ((#v-server-snapshotagent-enabled)) (`boolean: false`) - If true, the chart will install resources necessary to run the snapshot agent. - `interval` ((#v-server-snapshotagent-interval)) (`string: 1h`) - Interval at which to perform snapshots. - Refer to [`interval`](https://developer.hashicorp.com/consul/commands/snapshot/agent#interval) + Refer to [`interval`](/consul/commands/snapshot/agent#interval) - `configSecret` ((#v-server-snapshotagent-configsecret)) - A Kubernetes or Vault secret that should be manually created to contain the entire config to be used on the snapshot agent. This is the preferred method of configuration since there are usually storage - credentials present. Please refer to the [Snapshot agent config](https://developer.hashicorp.com/consul/commands/snapshot/agent#config-file-options) + credentials present. Please refer to the [Snapshot agent config](/consul/commands/snapshot/agent#config-file-options) for details. - `secretName` ((#v-server-snapshotagent-configsecret-secretname)) (`string: null`) - The name of the Kubernetes secret or Vault secret path that holds the snapshot agent config. @@ -975,7 +975,7 @@ Use these links to navigate to a particular top-level stanza. - `k8sAuthMethodHost` ((#v-externalservers-k8sauthmethodhost)) (`string: null`) - If you are setting `global.acls.manageSystemACLs` and `connectInject.enabled` to true, set `k8sAuthMethodHost` to the address of the Kubernetes API server. This address must be reachable from the Consul servers. - Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](/consul/docs/security/acl/auth-methods/kubernetes). You could retrieve this value from your `kubeconfig` by running: @@ -998,7 +998,7 @@ Use these links to navigate to a particular top-level stanza. - `image` ((#v-client-image)) (`string: null`) - The name of the Docker image (including any tag) for the containers running Consul client agents. - - `join` ((#v-client-join)) (`array: null`) - A list of valid [`-retry-join` values](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_retry_join). + - `join` ((#v-client-join)) (`array: null`) - A list of valid [`-retry-join` values](/consul/docs/agent/config/cli-flags#_retry_join). If this is `null` (default), then the clients will attempt to automatically join the server cluster running within Kubernetes. This means that with `server.enabled` set to true, clients will automatically @@ -1019,7 +1019,7 @@ Use these links to navigate to a particular top-level stanza. required for Connect. - `nodeMeta` ((#v-client-nodemeta)) - nodeMeta specifies an arbitrary metadata key/value pair to associate with the node - (refer to [`-node-meta`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_node_meta)) + (refer to [`-node-meta`](/consul/docs/agent/config/cli-flags#_node_meta)) - `pod-name` ((#v-client-nodemeta-pod-name)) (`string: ${HOSTNAME}`) @@ -1063,7 +1063,7 @@ Use these links to navigate to a particular top-level stanza. - `tlsInit` ((#v-client-containersecuritycontext-tlsinit)) (`map`) - The tls-init initContainer - - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul + - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](/consul/docs/agent/config/config-files) for Consul clients. This will be saved as-is into a ConfigMap that is read by the Consul client agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -1329,16 +1329,16 @@ Use these links to navigate to a particular top-level stanza. will inherit from `global.metrics.enabled` value. - `provider` ((#v-ui-metrics-provider)) (`string: prometheus`) - Provider for metrics. Refer to - [`metrics_provider`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_metrics_provider) + [`metrics_provider`](/consul/docs/agent/config/config-files#ui_config_metrics_provider) This value is only used if `ui.enabled` is set to true. - `baseURL` ((#v-ui-metrics-baseurl)) (`string: http://prometheus-server`) - baseURL is the URL of the prometheus server, usually the service URL. This value is only used if `ui.enabled` is set to true. - - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to [`dashboard_url_templates`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates) + - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to [`dashboard_url_templates`](/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates) configuration. - - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets [`dashboardURLTemplates.service`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates_service). + - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets [`dashboardURLTemplates.service`](/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates_service). ### syncCatalog ((#h-synccatalog)) @@ -1358,7 +1358,7 @@ Use these links to navigate to a particular top-level stanza. to run the sync program. - `default` ((#v-synccatalog-default)) (`boolean: true`) - If true, all valid services in K8S are - synced by default. If false, the service must be [annotated](https://developer.hashicorp.com/consul/docs/k8s/service-sync#enable-and-disable-sync) + synced by default. If false, the service must be [annotated](/consul/docs/k8s/service-sync#enable-and-disable-sync) properly to sync. In either case an annotation can override the default. @@ -1538,7 +1538,7 @@ Use these links to navigate to a particular top-level stanza. - `default` ((#v-connectinject-default)) (`boolean: false`) - If true, the injector will inject the Connect sidecar into all pods by default. Otherwise, pods must specify the - [injection annotation](https://developer.hashicorp.com/consul/docs/k8s/connect#consul-hashicorp-com-connect-inject) + [injection annotation](/consul/docs/k8s/connect#consul-hashicorp-com-connect-inject) to opt-in to Connect injection. If this is true, pods can use the same annotation to explicitly opt-out of injection. @@ -1816,8 +1816,8 @@ Use these links to navigate to a particular top-level stanza. If set to an empty string all service accounts can log in. This only has effect if ACLs are enabled. - Refer to Auth methods [Binding rules](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods#binding-rules) - and [Trusted identiy attributes](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes#trusted-identity-attributes) + Refer to Auth methods [Binding rules](/consul/docs/security/acl/auth-methods#binding-rules) + and [Trusted identiy attributes](/consul/docs/security/acl/auth-methods/kubernetes#trusted-identity-attributes) for more details. Requires Consul >= v1.5. @@ -1873,11 +1873,11 @@ Use these links to navigate to a particular top-level stanza. ### meshGateway ((#h-meshgateway)) -- `meshGateway` ((#v-meshgateway)) - [Mesh Gateways](https://developer.hashicorp.com/consul/docs/connect/gateways/mesh-gateway) enable Consul Connect to work across Consul datacenters. +- `meshGateway` ((#v-meshgateway)) - [Mesh Gateways](/consul/docs/connect/gateways/mesh-gateway) enable Consul Connect to work across Consul datacenters. - - `enabled` ((#v-meshgateway-enabled)) (`boolean: false`) - If [mesh gateways](https://developer.hashicorp.com/consul/docs/connect/gateways/mesh-gateway) are enabled, a Deployment will be created that runs + - `enabled` ((#v-meshgateway-enabled)) (`boolean: false`) - If [mesh gateways](/consul/docs/connect/gateways/mesh-gateway) are enabled, a Deployment will be created that runs gateways and Consul Connect will be configured to use gateways. - This setting is required for [Cluster Peering](https://developer.hashicorp.com/consul/docs/connect/cluster-peering/k8s). + This setting is required for [Cluster Peering](/consul/docs/connect/cluster-peering/k8s). Requirements: consul 1.6.0+ if using `global.acls.manageSystemACLs``. - `replicas` ((#v-meshgateway-replicas)) (`integer: 1`) - Number of replicas for the Deployment. From d49d0683cfec70065d142264eb616bec4d7a13ae Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 21 Mar 2023 20:00:21 -0700 Subject: [PATCH 122/421] backport of commit 8f4a326d850d5652e84dfe83505a8ef1e40190d2 (#16707) Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> Co-authored-by: David Yu --- website/content/docs/k8s/helm.mdx | 60 +++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/website/content/docs/k8s/helm.mdx b/website/content/docs/k8s/helm.mdx index 69614cd2ee97..ac6d802c12d5 100644 --- a/website/content/docs/k8s/helm.mdx +++ b/website/content/docs/k8s/helm.mdx @@ -373,14 +373,20 @@ Use these links to navigate to a particular top-level stanza. for all Consul and consul-k8s-control-plane components. This requires Consul >= 1.4. - - `bootstrapToken` ((#v-global-acls-bootstraptoken)) - A Kubernetes or Vault secret containing the bootstrap token to use for - creating policies and tokens for all Consul and consul-k8s-control-plane components. - If set, we will skip ACL bootstrapping of the servers and will only - initialize ACLs for the Consul clients and consul-k8s-control-plane system components. + - `bootstrapToken` ((#v-global-acls-bootstraptoken)) - A Kubernetes or Vault secret containing the bootstrap token to use for creating policies and + tokens for all Consul and consul-k8s-control-plane components. If `secretName` and `secretKey` + are unset, a default secret name and secret key are used. If the secret is populated, then + we will skip ACL bootstrapping of the servers and will only initialize ACLs for the Consul + clients and consul-k8s-control-plane system components. + If the secret is empty, then we will bootstrap ACLs on the Consul servers, and write the + bootstrap token to this secret. If ACLs are already bootstrapped on the servers, then the + secret must contain the bootstrap token. - `secretName` ((#v-global-acls-bootstraptoken-secretname)) (`string: null`) - The name of the Kubernetes or Vault secret that holds the bootstrap token. + If unset, this defaults to `{{ global.name }}-bootstrap-acl-token`. - `secretKey` ((#v-global-acls-bootstraptoken-secretkey)) (`string: null`) - The key within the Kubernetes or Vault secret that holds the bootstrap token. + If unset, this defaults to `token`. - `createReplicationToken` ((#v-global-acls-createreplicationtoken)) (`boolean: false`) - If true, an ACL token will be created that can be used in secondary datacenters for replication. This should only be set to true in the @@ -1718,7 +1724,19 @@ Use these links to navigate to a particular top-level stanza. "sample/annotation2": "bar" ``` - - `resources` ((#v-connectinject-resources)) (`map`) - The resource settings for connect inject pods. + - `resources` ((#v-connectinject-resources)) (`map`) - The resource settings for connect inject pods. The defaults, are optimized for getting started worklows on developer deployments. The settings should be tweaked for production deployments. + + - `requests` ((#v-connectinject-resources-requests)) + + - `memory` ((#v-connectinject-resources-requests-memory)) (`string: 50Mi`) - Recommended production default: 500Mi + + - `cpu` ((#v-connectinject-resources-requests-cpu)) (`string: 50m`) - Recommended production default: 250m + + - `limits` ((#v-connectinject-resources-limits)) + + - `memory` ((#v-connectinject-resources-limits-memory)) (`string: 50Mi`) - Recommended production default: 500Mi + + - `cpu` ((#v-connectinject-resources-limits-cpu)) (`string: 50m`) - Recommended production default: 250m - `failurePolicy` ((#v-connectinject-failurepolicy)) (`string: Fail`) - Sets the failurePolicy for the mutating webhook. By default this will cause pods not part of the consul installation to fail scheduling while the webhook is offline. This prevents a pod from skipping mutation if the webhook were to be momentarily offline. @@ -1859,17 +1877,33 @@ Use these links to navigate to a particular top-level stanza. - `requests` ((#v-connectinject-sidecarproxy-resources-requests)) - - `memory` ((#v-connectinject-sidecarproxy-resources-requests-memory)) (`string: null`) - Recommended default: 100Mi + - `memory` ((#v-connectinject-sidecarproxy-resources-requests-memory)) (`string: null`) - Recommended production default: 100Mi - - `cpu` ((#v-connectinject-sidecarproxy-resources-requests-cpu)) (`string: null`) - Recommended default: 100m + - `cpu` ((#v-connectinject-sidecarproxy-resources-requests-cpu)) (`string: null`) - Recommended production default: 100m - `limits` ((#v-connectinject-sidecarproxy-resources-limits)) - - `memory` ((#v-connectinject-sidecarproxy-resources-limits-memory)) (`string: null`) - Recommended default: 100Mi + - `memory` ((#v-connectinject-sidecarproxy-resources-limits-memory)) (`string: null`) - Recommended production default: 100Mi + + - `cpu` ((#v-connectinject-sidecarproxy-resources-limits-cpu)) (`string: null`) - Recommended production default: 100m + + - `initContainer` ((#v-connectinject-initcontainer)) (`map`) - The resource settings for the Connect injected init container. If null, the resources + won't be set for the initContainer. The defaults are optimized for developer instances of + Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times. + + - `resources` ((#v-connectinject-initcontainer-resources)) + + - `requests` ((#v-connectinject-initcontainer-resources-requests)) + + - `memory` ((#v-connectinject-initcontainer-resources-requests-memory)) (`string: 25Mi`) - Recommended production default: 150Mi + + - `cpu` ((#v-connectinject-initcontainer-resources-requests-cpu)) (`string: 50m`) - Recommended production default: 250m + + - `limits` ((#v-connectinject-initcontainer-resources-limits)) - - `cpu` ((#v-connectinject-sidecarproxy-resources-limits-cpu)) (`string: null`) - Recommended default: 100m + - `memory` ((#v-connectinject-initcontainer-resources-limits-memory)) (`string: 150Mi`) - Recommended production default: 150Mi - - `initContainer` ((#v-connectinject-initcontainer)) (`map`) - The resource settings for the Connect injected init container. + - `cpu` ((#v-connectinject-initcontainer-resources-limits-cpu)) (`string: null`) - Recommended production default: 500m ### meshGateway ((#h-meshgateway)) @@ -2041,8 +2075,7 @@ Use these links to navigate to a particular top-level stanza. for a specific gateway. Requirements: consul >= 1.8.0 - - `enabled` ((#v-ingressgateways-enabled)) (`boolean: false`) - Enable ingress gateway deployment. Requires `connectInject.enabled=true` - and `client.enabled=true`. + - `enabled` ((#v-ingressgateways-enabled)) (`boolean: false`) - Enable ingress gateway deployment. Requires `connectInject.enabled=true`. - `defaults` ((#v-ingressgateways-defaults)) - Defaults sets default values for all gateway fields. With the exception of annotations, defining any of these values in the `gateways` list @@ -2171,8 +2204,7 @@ Use these links to navigate to a particular top-level stanza. for a specific gateway. Requirements: consul >= 1.8.0 - - `enabled` ((#v-terminatinggateways-enabled)) (`boolean: false`) - Enable terminating gateway deployment. Requires `connectInject.enabled=true` - and `client.enabled=true`. + - `enabled` ((#v-terminatinggateways-enabled)) (`boolean: false`) - Enable terminating gateway deployment. Requires `connectInject.enabled=true`. - `defaults` ((#v-terminatinggateways-defaults)) - Defaults sets default values for all gateway fields. With the exception of annotations, defining any of these values in the `gateways` list From 83d26bc21f348dbd7686d45a1032581c7a8c572a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Mar 2023 06:50:49 -0700 Subject: [PATCH 123/421] backport of commit 746a0a1d736fd7259dd664ebbecef00fe44c9a7b (#16735) Co-authored-by: Eric --- .changelog/16729.txt | 3 ++ agent/consul/prepared_query_endpoint.go | 6 +-- agent/consul/prepared_query_endpoint_test.go | 41 ++++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 .changelog/16729.txt diff --git a/.changelog/16729.txt b/.changelog/16729.txt new file mode 100644 index 000000000000..36c6e1aeabb2 --- /dev/null +++ b/.changelog/16729.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue resulting in prepared query failover to cluster peers never un-failing over. +``` diff --git a/agent/consul/prepared_query_endpoint.go b/agent/consul/prepared_query_endpoint.go index ffa4b5e50915..54ef82bafdd1 100644 --- a/agent/consul/prepared_query_endpoint.go +++ b/agent/consul/prepared_query_endpoint.go @@ -468,7 +468,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, // by the query setup. if len(reply.Nodes) == 0 { wrapper := &queryServerWrapper{srv: p.srv, executeRemote: p.ExecuteRemote} - if err := queryFailover(wrapper, query, args, reply); err != nil { + if err := queryFailover(wrapper, *query, args, reply); err != nil { return err } } @@ -707,7 +707,7 @@ func (q *queryServerWrapper) GetOtherDatacentersByDistance() ([]string, error) { // queryFailover runs an algorithm to determine which DCs to try and then calls // them to try to locate alternative services. -func queryFailover(q queryServer, query *structs.PreparedQuery, +func queryFailover(q queryServer, query structs.PreparedQuery, args *structs.PreparedQueryExecuteRequest, reply *structs.PreparedQueryExecuteResponse) error { @@ -789,7 +789,7 @@ func queryFailover(q queryServer, query *structs.PreparedQuery, // the remote query as well. remote := &structs.PreparedQueryExecuteRemoteRequest{ Datacenter: dc, - Query: *query, + Query: query, Limit: args.Limit, QueryOptions: args.QueryOptions, Connect: args.Connect, diff --git a/agent/consul/prepared_query_endpoint_test.go b/agent/consul/prepared_query_endpoint_test.go index 418710b8b50d..ca459ddfd896 100644 --- a/agent/consul/prepared_query_endpoint_test.go +++ b/agent/consul/prepared_query_endpoint_test.go @@ -2092,16 +2092,16 @@ func TestPreparedQuery_Execute(t *testing.T) { require.NoError(t, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID)) // Update the health of a node to mark it critical. - setHealth := func(t *testing.T, codec rpc.ClientCodec, dc string, node string, health string) { + setHealth := func(t *testing.T, codec rpc.ClientCodec, dc string, i int, health string) { t.Helper() req := structs.RegisterRequest{ Datacenter: dc, - Node: node, + Node: fmt.Sprintf("node%d", i), Address: "127.0.0.1", Service: &structs.NodeService{ Service: "foo", Port: 8000, - Tags: []string{"dc1", "tag1"}, + Tags: []string{dc, fmt.Sprintf("tag%d", i)}, }, Check: &structs.HealthCheck{ Name: "failing", @@ -2113,7 +2113,7 @@ func TestPreparedQuery_Execute(t *testing.T) { var reply struct{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply)) } - setHealth(t, codec1, "dc1", "node1", api.HealthCritical) + setHealth(t, codec1, "dc1", 1, api.HealthCritical) // The failing node should be filtered. t.Run("failing node filtered", func(t *testing.T) { @@ -2133,7 +2133,7 @@ func TestPreparedQuery_Execute(t *testing.T) { }) // Upgrade it to a warning and re-query, should be 10 nodes again. - setHealth(t, codec1, "dc1", "node1", api.HealthWarning) + setHealth(t, codec1, "dc1", 1, api.HealthWarning) t.Run("warning nodes are included", func(t *testing.T) { req := structs.PreparedQueryExecuteRequest{ Datacenter: "dc1", @@ -2303,7 +2303,7 @@ func TestPreparedQuery_Execute(t *testing.T) { // Now fail everything in dc1 and we should get an empty list back. for i := 0; i < 10; i++ { - setHealth(t, codec1, "dc1", fmt.Sprintf("node%d", i+1), api.HealthCritical) + setHealth(t, codec1, "dc1", i+1, api.HealthCritical) } t.Run("everything is failing so should get empty list", func(t *testing.T) { req := structs.PreparedQueryExecuteRequest{ @@ -2474,7 +2474,7 @@ func TestPreparedQuery_Execute(t *testing.T) { // Set all checks in dc2 as critical for i := 0; i < 10; i++ { - setHealth(t, codec2, "dc2", fmt.Sprintf("node%d", i+1), api.HealthCritical) + setHealth(t, codec2, "dc2", i+1, api.HealthCritical) } // Now we should see 9 nodes from dc3 (we have the tag filter still) @@ -2493,6 +2493,31 @@ func TestPreparedQuery_Execute(t *testing.T) { } expectFailoverPeerNodes(t, &query, &reply, 9) }) + + // Set all checks in dc1 as passing + for i := 0; i < 10; i++ { + setHealth(t, codec1, "dc1", i+1, api.HealthPassing) + } + + // Nothing is healthy so nothing is returned + t.Run("un-failing over", func(t *testing.T) { + retry.Run(t, func(r *retry.R) { + req := structs.PreparedQueryExecuteRequest{ + Datacenter: "dc1", + QueryIDOrName: query.Query.ID, + QueryOptions: structs.QueryOptions{Token: execToken}, + } + + var reply structs.PreparedQueryExecuteResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply)) + + for _, node := range reply.Nodes { + assert.NotEqual(t, "node3", node.Node.Node) + } + + expectNodes(t, &query, &reply, 9) + }) + }) } func TestPreparedQuery_Execute_ForwardLeader(t *testing.T) { @@ -2982,7 +3007,7 @@ func (m *mockQueryServer) ExecuteRemote(args *structs.PreparedQueryExecuteRemote func TestPreparedQuery_queryFailover(t *testing.T) { t.Parallel() - query := &structs.PreparedQuery{ + query := structs.PreparedQuery{ Name: "test", Service: structs.ServiceQuery{ Failover: structs.QueryFailoverOptions{ From 9d636aa81677b955eb339d1d34745501699289cb Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 22 Mar 2023 13:51:58 -0700 Subject: [PATCH 124/421] Backport of Refactor xDS tests into release/1.15.x (#16741) * backport of commit ed4a98f30010fef0998b1feccf7d033d5c99a5bb * add merge function --------- Co-authored-by: Eric --- agent/agent_endpoint_test.go | 4 +- agent/proxycfg/manager_test.go | 2 +- agent/proxycfg/testing.go | 14 +- agent/proxycfg/testing_api_gateway.go | 2 +- agent/proxycfg/testing_connect_proxy.go | 14 +- agent/proxycfg/testing_ingress_gateway.go | 2 +- agent/proxycfg/testing_upstreams.go | 243 +++++++++++------- agent/proxycfg_test.go | 2 +- agent/sidecar_service_test.go | 4 +- agent/structs/discovery_chain.go | 34 ++- agent/structs/testing_connect_proxy_config.go | 57 ++-- agent/xds/clusters_test.go | 209 ++++++++------- agent/xds/delta_envoy_extender_oss_test.go | 14 +- agent/xds/endpoints_test.go | 119 +++++---- .../runtime_config_oss_test.go | 6 +- agent/xds/golden_test.go | 33 ++- agent/xds/listeners_test.go | 232 +++++++++-------- agent/xds/resources_test.go | 18 +- agent/xds/routes_test.go | 49 ++-- ...and-redirect-to-cluster-peer.latest.golden | 66 ++--- ...and-failover-to-cluster-peer.latest.golden | 136 +++++----- ...and-redirect-to-cluster-peer.latest.golden | 136 +++++----- .../validateupstream_test.go | 18 +- agent/xds/xds_protocol_helpers_test.go | 2 +- command/services/config_test.go | 8 +- 25 files changed, 791 insertions(+), 633 deletions(-) diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index c9cfbee45cb8..2c5ee9f3727a 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -186,7 +186,7 @@ func TestAgent_Services_ExternalConnectProxy(t *testing.T) { Port: 5000, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "db", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, } a.State.AddServiceWithChecks(srv1, nil, "", false) @@ -226,7 +226,7 @@ func TestAgent_Services_Sidecar(t *testing.T) { LocallyRegisteredAsSidecar: true, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "db", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), Mode: structs.ProxyModeTransparent, TransparentProxy: structs.TransparentProxyConfig{ OutboundListenerPort: 10101, diff --git a/agent/proxycfg/manager_test.go b/agent/proxycfg/manager_test.go index 7d946ce823f1..cad98796f737 100644 --- a/agent/proxycfg/manager_test.go +++ b/agent/proxycfg/manager_test.go @@ -95,7 +95,7 @@ func TestManager_BasicLifecycle(t *testing.T) { }) } - upstreams := structs.TestUpstreams(t) + upstreams := structs.TestUpstreams(t, false) for i := range upstreams { upstreams[i].DestinationNamespace = structs.IntentionDefaultNamespace upstreams[i].DestinationPartition = api.PartitionDefaultName diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 8fbb247e084c..7c4b85b97124 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -552,7 +552,7 @@ func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "bar", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -573,7 +573,7 @@ func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "bar", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -594,7 +594,7 @@ func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "bar", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -620,7 +620,7 @@ func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "foo", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -641,7 +641,7 @@ func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "foo", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -662,7 +662,7 @@ func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "foo", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -683,7 +683,7 @@ func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes { }, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "foo", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, Checks: structs.HealthChecks{ diff --git a/agent/proxycfg/testing_api_gateway.go b/agent/proxycfg/testing_api_gateway.go index dd55f2eec5c2..b63c624e05a9 100644 --- a/agent/proxycfg/testing_api_gateway.go +++ b/agent/proxycfg/testing_api_gateway.go @@ -105,7 +105,7 @@ func TestConfigSnapshotAPIGateway( }) } - upstreams := structs.TestUpstreams(t) + upstreams := structs.TestUpstreams(t, false) baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot( t, variation, upstreams, additionalEntries..., diff --git a/agent/proxycfg/testing_connect_proxy.go b/agent/proxycfg/testing_connect_proxy.go index 394687a44bcc..332c8ef2d8f3 100644 --- a/agent/proxycfg/testing_connect_proxy.go +++ b/agent/proxycfg/testing_connect_proxy.go @@ -7,6 +7,7 @@ import ( "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/assert" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" @@ -23,7 +24,7 @@ func TestConfigSnapshot(t testing.T, nsFn func(ns *structs.NodeService), extraUp assert.True(t, dbChain.Default) var ( - upstreams = structs.TestUpstreams(t) + upstreams = structs.TestUpstreams(t, false) dbUpstream = upstreams[0] geoUpstream = upstreams[1] @@ -93,19 +94,25 @@ func TestConfigSnapshot(t testing.T, nsFn func(ns *structs.NodeService), extraUp func TestConfigSnapshotDiscoveryChain( t testing.T, variation string, + enterprise bool, nsFn func(ns *structs.NodeService), extraUpdates []UpdateEvent, additionalEntries ...structs.ConfigEntry, ) *ConfigSnapshot { roots, leaf := TestCerts(t) + var entMeta acl.EnterpriseMeta + if enterprise { + entMeta = acl.NewEnterpriseMetaWithPartition("ap1", "ns1") + } + var ( - upstreams = structs.TestUpstreams(t) + upstreams = structs.TestUpstreams(t, enterprise) geoUpstream = upstreams[1] geoUID = NewUpstreamID(&geoUpstream) - webSN = structs.ServiceIDString("web", nil) + webSN = structs.ServiceIDString("web", &entMeta) ) baseEvents := testSpliceEvents([]UpdateEvent{ @@ -157,6 +164,7 @@ func TestConfigSnapshotDiscoveryChain( }, Meta: nil, TaggedAddresses: nil, + EnterpriseMeta: entMeta, }, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates)) } diff --git a/agent/proxycfg/testing_ingress_gateway.go b/agent/proxycfg/testing_ingress_gateway.go index bca283f18505..274f9931b75e 100644 --- a/agent/proxycfg/testing_ingress_gateway.go +++ b/agent/proxycfg/testing_ingress_gateway.go @@ -84,7 +84,7 @@ func TestConfigSnapshotIngressGateway( }, }}) - upstreams := structs.TestUpstreams(t) + upstreams := structs.TestUpstreams(t, false) upstreams = structs.Upstreams{upstreams[0]} // just keep 'db' baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot( diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index 3bc09048acf8..0e09aff9e104 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -5,6 +5,7 @@ import ( "github.com/mitchellh/go-testing-interface" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" @@ -23,7 +24,7 @@ func setupTestVariationConfigEntriesAndSnapshot( dbUID = NewUpstreamID(&dbUpstream) ) - dbChain := setupTestVariationDiscoveryChain(t, variation, additionalEntries...) + dbChain := setupTestVariationDiscoveryChain(t, variation, dbUID.EnterpriseMeta, additionalEntries...) nodes := TestUpstreamNodes(t, "db") if variation == "register-to-terminating-gateway" { @@ -46,29 +47,42 @@ func setupTestVariationConfigEntriesAndSnapshot( }, } + dbOpts := structs.DiscoveryTargetOpts{ + Service: dbUID.Name, + Namespace: dbUID.NamespaceOrDefault(), + Partition: dbUID.PartitionOrDefault(), + Datacenter: "dc1", + } + dbChainID := structs.ChainID(dbOpts) + makeChainID := func(opts structs.DiscoveryTargetOpts) string { + return structs.ChainID(structs.MergeDiscoveryTargetOpts(dbOpts, opts)) + } + switch variation { case "default": case "simple-with-overrides": case "simple": case "external-sni": case "failover": + chainID := makeChainID(structs.DiscoveryTargetOpts{Service: "fail"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:fail.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + chainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesAlternate(t), }, }) case "failover-through-remote-gateway-triggered": events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + dbChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesInStatus(t, "critical"), }, }) fallthrough case "failover-through-remote-gateway": + chainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc2"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc2:" + dbUID.String(), + CorrelationID: "upstream-target:" + chainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesDC2(t), }, @@ -91,8 +105,13 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }, }) + uid := UpstreamID{ + Name: "db", + Peer: "cluster-01", + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), ""), + } events = append(events, UpdateEvent{ - CorrelationID: "upstream-peer:db?peer=cluster-01", + CorrelationID: "upstream-peer:" + uid.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "cluster-01", "10.40.1.1", false)}, }, @@ -109,83 +128,93 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }, }) + uid := UpstreamID{ + Name: "db", + Peer: "cluster-01", + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), ""), + } events = append(events, UpdateEvent{ - CorrelationID: "upstream-peer:db?peer=cluster-01", + CorrelationID: "upstream-peer:" + uid.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc2", "cluster-01", "10.40.1.1", false)}, }, }) case "failover-through-double-remote-gateway-triggered": + chainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc2"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + dbChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesInStatus(t, "critical"), }, - }) - events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc2:" + dbUID.String(), - Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestUpstreamNodesInStatusDC2(t, "critical"), - }, - }) + }, + UpdateEvent{ + CorrelationID: "upstream-target:" + chainID + ":" + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestUpstreamNodesInStatusDC2(t, "critical"), + }, + }) fallthrough case "failover-through-double-remote-gateway": + chainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc3"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc3:" + dbUID.String(), + CorrelationID: "upstream-target:" + chainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesDC2(t), }, - }) - events = append(events, UpdateEvent{ - CorrelationID: "mesh-gateway:dc2:" + dbUID.String(), - Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestGatewayNodesDC2(t), - }, - }) - events = append(events, UpdateEvent{ - CorrelationID: "mesh-gateway:dc3:" + dbUID.String(), - Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestGatewayNodesDC3(t), + }, + UpdateEvent{ + CorrelationID: "mesh-gateway:dc2:" + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestGatewayNodesDC2(t), + }, }, - }) + UpdateEvent{ + CorrelationID: "mesh-gateway:dc3:" + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestGatewayNodesDC3(t), + }, + }) case "failover-through-local-gateway-triggered": events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + dbChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesInStatus(t, "critical"), }, }) fallthrough case "failover-through-local-gateway": + chainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc2"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc2:" + dbUID.String(), + CorrelationID: "upstream-target:" + chainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesDC2(t), }, - }) - events = append(events, UpdateEvent{ - CorrelationID: "mesh-gateway:dc1:" + dbUID.String(), - Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestGatewayNodesDC1(t), - }, - }) + }, + UpdateEvent{ + CorrelationID: "mesh-gateway:dc1:" + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestGatewayNodesDC1(t), + }, + }) case "failover-through-double-local-gateway-triggered": + db2ChainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc2"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + dbChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesInStatus(t, "critical"), }, - }) - events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc2:" + dbUID.String(), - Result: &structs.IndexedCheckServiceNodes{ - Nodes: TestUpstreamNodesInStatusDC2(t, "critical"), - }, - }) + }, + UpdateEvent{ + CorrelationID: "upstream-target:" + db2ChainID + ":" + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: TestUpstreamNodesInStatusDC2(t, "critical"), + }, + }) fallthrough case "failover-through-double-local-gateway": + db3ChainID := makeChainID(structs.DiscoveryTargetOpts{Datacenter: "dc3"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:db.default.default.dc3:" + dbUID.String(), + CorrelationID: "upstream-target:" + db3ChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesDC2(t), }, @@ -197,14 +226,16 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }) case "splitter-with-resolver-redirect-multidc": + v1ChainID := makeChainID(structs.DiscoveryTargetOpts{ServiceSubset: "v1"}) + v2ChainID := makeChainID(structs.DiscoveryTargetOpts{ServiceSubset: "v2", Datacenter: "dc2"}) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:v1.db.default.default.dc1:" + dbUID.String(), + CorrelationID: "upstream-target:" + v1ChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodes(t, "db"), }, }) events = append(events, UpdateEvent{ - CorrelationID: "upstream-target:v2.db.default.default.dc2:" + dbUID.String(), + CorrelationID: "upstream-target:" + v2ChainID + ":" + dbUID.String(), Result: &structs.IndexedCheckServiceNodes{ Nodes: TestUpstreamNodesDC2(t), }, @@ -225,6 +256,7 @@ func setupTestVariationConfigEntriesAndSnapshot( func setupTestVariationDiscoveryChain( t testing.T, variation string, + entMeta acl.EnterpriseMeta, additionalEntries ...structs.ConfigEntry, ) *structs.CompiledDiscoveryChain { // Compile a chain. @@ -249,6 +281,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, }, @@ -256,13 +289,15 @@ func setupTestVariationDiscoveryChain( case "external-sni": entries = append(entries, &structs.ServiceConfigEntry{ - Kind: structs.ServiceDefaults, - Name: "db", - ExternalSNI: "db.some.other.service.mesh", + Kind: structs.ServiceDefaults, + Name: "db", + EnterpriseMeta: entMeta, + ExternalSNI: "db.some.other.service.mesh", }, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, }, @@ -272,6 +307,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -286,8 +322,9 @@ func setupTestVariationDiscoveryChain( case "failover-through-remote-gateway": entries = append(entries, &structs.ServiceConfigEntry{ - Kind: structs.ServiceDefaults, - Name: "db", + Kind: structs.ServiceDefaults, + Name: "db", + EnterpriseMeta: entMeta, MeshGateway: structs.MeshGatewayConfig{ Mode: structs.MeshGatewayModeRemote, }, @@ -295,6 +332,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -309,6 +347,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -325,6 +364,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Redirect: &structs.ServiceResolverRedirect{ @@ -337,8 +377,9 @@ func setupTestVariationDiscoveryChain( case "failover-through-double-remote-gateway": entries = append(entries, &structs.ServiceConfigEntry{ - Kind: structs.ServiceDefaults, - Name: "db", + Kind: structs.ServiceDefaults, + Name: "db", + EnterpriseMeta: entMeta, MeshGateway: structs.MeshGatewayConfig{ Mode: structs.MeshGatewayModeRemote, }, @@ -346,6 +387,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -360,8 +402,9 @@ func setupTestVariationDiscoveryChain( case "failover-through-local-gateway": entries = append(entries, &structs.ServiceConfigEntry{ - Kind: structs.ServiceDefaults, - Name: "db", + Kind: structs.ServiceDefaults, + Name: "db", + EnterpriseMeta: entMeta, MeshGateway: structs.MeshGatewayConfig{ Mode: structs.MeshGatewayModeLocal, }, @@ -369,6 +412,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -383,8 +427,9 @@ func setupTestVariationDiscoveryChain( case "failover-through-double-local-gateway": entries = append(entries, &structs.ServiceConfigEntry{ - Kind: structs.ServiceDefaults, - Name: "db", + Kind: structs.ServiceDefaults, + Name: "db", + EnterpriseMeta: entMeta, MeshGateway: structs.MeshGatewayConfig{ Mode: structs.MeshGatewayModeLocal, }, @@ -392,6 +437,7 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ @@ -402,25 +448,29 @@ func setupTestVariationDiscoveryChain( }, ) case "splitter-with-resolver-redirect-multidc": + em := acl.NewEnterpriseMetaWithPartition(entMeta.PartitionOrDefault(), acl.NamespaceOrDefault("")) entries = append(entries, &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + EnterpriseMeta: em, Config: map[string]interface{}{ "protocol": "http", }, }, &structs.ServiceSplitterConfigEntry{ - Kind: structs.ServiceResolver, - Name: "db", + Kind: structs.ServiceResolver, + Name: "db", + EnterpriseMeta: entMeta, Splits: []structs.ServiceSplit{ {Weight: 50, Service: "db-dc1"}, {Weight: 50, Service: "db-dc2"}, }, }, &structs.ServiceResolverConfigEntry{ - Kind: structs.ServiceResolver, - Name: "db-dc1", + Kind: structs.ServiceResolver, + Name: "db-dc1", + EnterpriseMeta: entMeta, Redirect: &structs.ServiceResolverRedirect{ Service: "db", ServiceSubset: "v1", @@ -428,8 +478,9 @@ func setupTestVariationDiscoveryChain( }, }, &structs.ServiceResolverConfigEntry{ - Kind: structs.ServiceResolver, - Name: "db-dc2", + Kind: structs.ServiceResolver, + Name: "db-dc2", + EnterpriseMeta: entMeta, Redirect: &structs.ServiceResolverRedirect{ Service: "db", ServiceSubset: "v2", @@ -437,8 +488,9 @@ func setupTestVariationDiscoveryChain( }, }, &structs.ServiceResolverConfigEntry{ - Kind: structs.ServiceResolver, - Name: "db", + Kind: structs.ServiceResolver, + Name: "db", + EnterpriseMeta: entMeta, Subsets: map[string]structs.ServiceResolverSubset{ "v1": { Filter: "Service.Meta.version == v1", @@ -454,19 +506,22 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + EnterpriseMeta: entMeta, Config: map[string]interface{}{ "protocol": "http", }, }, &structs.ServiceSplitterConfigEntry{ - Kind: structs.ServiceSplitter, - Name: "db", + Kind: structs.ServiceSplitter, + Name: "db", + EnterpriseMeta: entMeta, Splits: []structs.ServiceSplit{ { Weight: 95.5, @@ -506,19 +561,22 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + EnterpriseMeta: entMeta, Config: map[string]interface{}{ "protocol": "grpc", }, }, &structs.ServiceRouterConfigEntry{ - Kind: structs.ServiceRouter, - Name: "db", + Kind: structs.ServiceRouter, + Name: "db", + EnterpriseMeta: entMeta, Routes: []structs.ServiceRoute{ { Match: &structs.ServiceRouteMatch{ @@ -538,19 +596,22 @@ func setupTestVariationDiscoveryChain( &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, Name: "db", + EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, }, &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + EnterpriseMeta: entMeta, Config: map[string]interface{}{ "protocol": "http", }, }, &structs.ServiceSplitterConfigEntry{ - Kind: structs.ServiceSplitter, - Name: "split-3-ways", + Kind: structs.ServiceSplitter, + Name: "split-3-ways", + EnterpriseMeta: entMeta, Splits: []structs.ServiceSplit{ {Weight: 95.5, Service: "big-side"}, {Weight: 4, Service: "goldilocks-side"}, @@ -558,8 +619,9 @@ func setupTestVariationDiscoveryChain( }, }, &structs.ServiceRouterConfigEntry{ - Kind: structs.ServiceRouter, - Name: "db", + Kind: structs.ServiceRouter, + Name: "db", + EnterpriseMeta: entMeta, Routes: []structs.ServiceRoute{ { Match: httpMatch(&structs.ServiceRouteHTTPMatch{ @@ -790,23 +852,26 @@ func setupTestVariationDiscoveryChain( case "lb-resolver": entries = append(entries, &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + EnterpriseMeta: entMeta, Config: map[string]interface{}{ "protocol": "http", }, }, &structs.ServiceSplitterConfigEntry{ - Kind: structs.ServiceSplitter, - Name: "db", + Kind: structs.ServiceSplitter, + Name: "db", + EnterpriseMeta: entMeta, Splits: []structs.ServiceSplit{ {Weight: 95.5, Service: "something-else"}, {Weight: 4.5, Service: "db"}, }, }, &structs.ServiceResolverConfigEntry{ - Kind: structs.ServiceResolver, - Name: "db", + Kind: structs.ServiceResolver, + Name: "db", + EnterpriseMeta: entMeta, LoadBalancer: &structs.LoadBalancer{ Policy: "ring_hash", RingHashConfig: &structs.RingHashConfig{ @@ -845,7 +910,7 @@ func setupTestVariationDiscoveryChain( entries = append(entries, additionalEntries...) } - return discoverychain.TestCompileConfigEntries(t, "db", "default", "default", "dc1", connect.TestClusterID+".consul", compileSetup, entries...) + return discoverychain.TestCompileConfigEntries(t, "db", entMeta.NamespaceOrDefault(), entMeta.PartitionOrDefault(), "dc1", connect.TestClusterID+".consul", compileSetup, entries...) } func httpMatch(http *structs.ServiceRouteHTTPMatch) *structs.ServiceRouteMatch { diff --git a/agent/proxycfg_test.go b/agent/proxycfg_test.go index 18a5c586245c..11f9b8fe7955 100644 --- a/agent/proxycfg_test.go +++ b/agent/proxycfg_test.go @@ -41,7 +41,7 @@ func TestAgent_local_proxycfg(t *testing.T) { LocallyRegisteredAsSidecar: true, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "db", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), } diff --git a/agent/sidecar_service_test.go b/agent/sidecar_service_test.go index 3b4a018a52d7..37854298e62e 100644 --- a/agent/sidecar_service_test.go +++ b/agent/sidecar_service_test.go @@ -93,7 +93,7 @@ func TestAgent_sidecarServiceFromNodeService(t *testing.T) { LocalServiceAddress: "127.0.127.0", LocalServicePort: 9999, Config: map[string]interface{}{"baz": "qux"}, - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, }, }, @@ -118,7 +118,7 @@ func TestAgent_sidecarServiceFromNodeService(t *testing.T) { LocalServiceAddress: "127.0.127.0", LocalServicePort: 9999, Config: map[string]interface{}{"baz": "qux"}, - Upstreams: structs.TestAddDefaultsToUpstreams(t, structs.TestUpstreams(t), + Upstreams: structs.TestAddDefaultsToUpstreams(t, structs.TestUpstreams(t, false), *structs.DefaultEnterpriseMetaInDefaultPartition()), }, }, diff --git a/agent/structs/discovery_chain.go b/agent/structs/discovery_chain.go index dc737e7479d6..cb2c1b6f1f98 100644 --- a/agent/structs/discovery_chain.go +++ b/agent/structs/discovery_chain.go @@ -59,7 +59,7 @@ type CompiledDiscoveryChain struct { // ID returns an ID that encodes the service, namespace, partition, and datacenter. // This ID allows us to compare a discovery chain target to the chain upstream itself. func (c *CompiledDiscoveryChain) ID() string { - return chainID(DiscoveryTargetOpts{ + return ChainID(DiscoveryTargetOpts{ Service: c.ServiceName, Namespace: c.Namespace, Partition: c.Partition, @@ -260,6 +260,34 @@ type DiscoveryTargetOpts struct { Peer string } +func MergeDiscoveryTargetOpts(o1 DiscoveryTargetOpts, o2 DiscoveryTargetOpts) DiscoveryTargetOpts { + if o2.Service != "" { + o1.Service = o2.Service + } + + if o2.ServiceSubset != "" { + o1.ServiceSubset = o2.ServiceSubset + } + + if o2.Namespace != "" { + o1.Namespace = o2.Namespace + } + + if o2.Partition != "" { + o1.Partition = o2.Partition + } + + if o2.Datacenter != "" { + o1.Datacenter = o2.Datacenter + } + + if o2.Peer != "" { + o1.Peer = o2.Peer + } + + return o1 +} + func NewDiscoveryTarget(opts DiscoveryTargetOpts) *DiscoveryTarget { t := &DiscoveryTarget{ Service: opts.Service, @@ -284,7 +312,7 @@ func (t *DiscoveryTarget) ToDiscoveryTargetOpts() DiscoveryTargetOpts { } } -func chainID(opts DiscoveryTargetOpts) string { +func ChainID(opts DiscoveryTargetOpts) string { // NOTE: this format is similar to the SNI syntax for simplicity if opts.Peer != "" { return fmt.Sprintf("%s.%s.default.external.%s", opts.Service, opts.Namespace, opts.Peer) @@ -296,7 +324,7 @@ func chainID(opts DiscoveryTargetOpts) string { } func (t *DiscoveryTarget) setID() { - t.ID = chainID(t.ToDiscoveryTargetOpts()) + t.ID = ChainID(t.ToDiscoveryTargetOpts()) } func (t *DiscoveryTarget) String() string { diff --git a/agent/structs/testing_connect_proxy_config.go b/agent/structs/testing_connect_proxy_config.go index fdee3f6937db..0021612e9e84 100644 --- a/agent/structs/testing_connect_proxy_config.go +++ b/agent/structs/testing_connect_proxy_config.go @@ -11,37 +11,46 @@ import ( func TestConnectProxyConfig(t testing.T) ConnectProxyConfig { return ConnectProxyConfig{ DestinationServiceName: "web", - Upstreams: TestUpstreams(t), + Upstreams: TestUpstreams(t, false), } } // TestUpstreams returns a set of upstreams to be used in tests exercising most // important configuration patterns. -func TestUpstreams(t testing.T) Upstreams { - return Upstreams{ - { - // We rely on this one having default type in a few tests... - DestinationName: "db", - LocalBindPort: 9191, - Config: map[string]interface{}{ - // Float because this is how it is decoded by JSON decoder so this - // enables the value returned to be compared directly to a decoded JSON - // response without spurious type loss. - "connect_timeout_ms": float64(1000), - }, - }, - { - DestinationType: UpstreamDestTypePreparedQuery, - DestinationName: "geo-cache", - LocalBindPort: 8181, - LocalBindAddress: "127.10.10.10", - }, - { - DestinationName: "upstream_socket", - LocalBindSocketPath: "/tmp/upstream.sock", - LocalBindSocketMode: "0700", +func TestUpstreams(t testing.T, enterprise bool) Upstreams { + db := Upstream{ + // We rely on this one having default type in a few tests... + DestinationName: "db", + LocalBindPort: 9191, + Config: map[string]interface{}{ + // Float because this is how it is decoded by JSON decoder so this + // enables the value returned to be compared directly to a decoded JSON + // response without spurious type loss. + "connect_timeout_ms": float64(1000), }, } + + geoCache := Upstream{ + DestinationType: UpstreamDestTypePreparedQuery, + DestinationName: "geo-cache", + LocalBindPort: 8181, + LocalBindAddress: "127.10.10.10", + } + + if enterprise { + db.DestinationNamespace = "foo" + db.DestinationPartition = "bar" + + geoCache.DestinationNamespace = "baz" + geoCache.DestinationPartition = "qux" + } + + return Upstreams{db, geoCache, { + DestinationName: "upstream_socket", + LocalBindSocketPath: "/tmp/upstream.sock", + LocalBindSocketMode: "0700", + }, + } } // TestAddDefaultsToUpstreams takes an array of upstreams (such as that from diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index e2269dc7ffa1..094b9ceb182d 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -21,6 +21,112 @@ import ( "github.com/hashicorp/consul/types" ) +type clusterTestCase struct { + name string + create func(t testinf.T) *proxycfg.ConfigSnapshot + overrideGoldenName string +} + +func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { + return []clusterTestCase{ + { + name: "custom-upstream-default-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", enterprise, func(ns *structs.NodeService) { + ns.Proxy.Upstreams[0].Config["envoy_cluster_json"] = + customAppClusterJSON(t, customClusterJSONOptions{ + Name: "myservice", + }) + }, nil) + }, + }, + { + name: "connect-proxy-with-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-chain-external-sni", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-chain-and-overrides", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-chain-and-failover", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) + }, + }, + { + name: "splitter-with-resolver-redirect", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-lb-in-resolver", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) + }, + }, + } +} + func TestClustersFromSnapshot(t *testing.T) { // TODO: we should move all of these to TestAllResourcesFromSnapshot // eventually to test all of the xDS types at once with the same input, @@ -29,11 +135,7 @@ func TestClustersFromSnapshot(t *testing.T) { t.Skip("too slow for testing.Short") } - tests := []struct { - name string - create func(t testinf.T) *proxycfg.ConfigSnapshot - overrideGoldenName string - }{ + tests := []clusterTestCase{ { name: "connect-proxy-with-tls-outgoing-min-version-auto", create: func(t testinf.T) *proxycfg.ConfigSnapshot { @@ -135,17 +237,6 @@ func TestClustersFromSnapshot(t *testing.T) { }, nil) }, }, - { - name: "custom-upstream-default-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", func(ns *structs.NodeService) { - ns.Proxy.Upstreams[0].Config["envoy_cluster_json"] = - customAppClusterJSON(t, customClusterJSONOptions{ - Name: "myservice", - }) - }, nil) - }, - }, { name: "custom-upstream-ignores-tls", overrideGoldenName: "custom-upstream", // should be the same @@ -245,90 +336,6 @@ func TestClustersFromSnapshot(t *testing.T) { }, nil) }, }, - { - name: "connect-proxy-with-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil) - }, - }, - { - name: "connect-proxy-with-chain-external-sni", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", nil, nil) - }, - }, - { - name: "connect-proxy-with-chain-and-overrides", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", nil, nil) - }, - }, - { - name: "connect-proxy-with-chain-and-failover", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", nil, nil) - }, - }, - { - name: "splitter-with-resolver-redirect", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", nil, nil) - }, - }, - { - name: "connect-proxy-lb-in-resolver", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", nil, nil) - }, - }, { name: "expose-paths-local-app-paths", create: func(t testinf.T) *proxycfg.ConfigSnapshot { @@ -767,6 +774,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, } + tests = append(tests, makeClusterDiscoChainTests(false)...) + latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { sf, err := xdscommon.DetermineSupportedProxyFeaturesFromString(envoyVersion) diff --git a/agent/xds/delta_envoy_extender_oss_test.go b/agent/xds/delta_envoy_extender_oss_test.go index 97a610fd4c53..3a8d1f17614b 100644 --- a/agent/xds/delta_envoy_extender_oss_test.go +++ b/agent/xds/delta_envoy_extender_oss_test.go @@ -92,7 +92,7 @@ end`, { name: "lambda-connect-proxy", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, makeLambdaServiceDefaults(false)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, makeLambdaServiceDefaults(false)) }, }, { @@ -107,13 +107,13 @@ end`, { name: "lambda-connect-proxy-with-terminating-gateway-upstream", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", nil, nil, makeLambdaServiceDefaults(false)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", false, nil, nil, makeLambdaServiceDefaults(false)) }, }, { name: "lambda-connect-proxy-opposite-meta", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, makeLambdaServiceDefaults(true)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, makeLambdaServiceDefaults(true)) }, }, { @@ -129,13 +129,13 @@ end`, { name: "lua-outbound-applies-to-upstreams", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, makeLuaServiceDefaults(false)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, makeLuaServiceDefaults(false)) }, }, { name: "lua-inbound-doesnt-applies-to-upstreams", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, makeLuaServiceDefaults(true)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, makeLuaServiceDefaults(true)) }, }, { @@ -183,7 +183,7 @@ end`, { name: "lua-connect-proxy-with-terminating-gateway-upstream", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", nil, nil, makeLambdaServiceDefaults(false)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", false, nil, nil, makeLambdaServiceDefaults(false)) }, }, { @@ -205,7 +205,7 @@ end`, }, } } - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nsFunc, nil, makeLambdaServiceDefaults(true)) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nsFunc, nil, makeLambdaServiceDefaults(true)) }, }, } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index e43865495a7c..31215adba93d 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -217,136 +217,143 @@ func Test_makeLoadAssignment(t *testing.T) { } } -func TestEndpointsFromSnapshot(t *testing.T) { - // TODO: we should move all of these to TestAllResourcesFromSnapshot - // eventually to test all of the xDS types at once with the same input, - // just as it would be triggered by our xDS server. - if testing.Short() { - t.Skip("too slow for testing.Short") - } +type endpointTestCase struct { + name string + create func(t testinf.T) *proxycfg.ConfigSnapshot + overrideGoldenName string +} - tests := []struct { - name string - create func(t testinf.T) *proxycfg.ConfigSnapshot - overrideGoldenName string - }{ +func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { + return []endpointTestCase{ { - name: "mesh-gateway", + name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, }, { - name: "mesh-gateway-using-federation-states", + name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, }, { - name: "mesh-gateway-newer-information-in-federation-states", + name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotMeshGateway(t, "newer-info-in-federation-states", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, }, { - name: "mesh-gateway-older-information-in-federation-states", + name: "connect-proxy-with-chain-and-failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotMeshGateway(t, "older-info-in-federation-states", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, }, { - name: "mesh-gateway-no-services", + name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-chain", + name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-chain-external-sni", + name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-chain-and-overrides", + name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-chain-and-failover", + name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", + name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", + name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", + name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", + name: "connect-proxy-with-default-chain-and-custom-cluster", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", enterprise, func(ns *structs.NodeService) { + ns.Proxy.Upstreams[0].Config["envoy_cluster_json"] = + customAppClusterJSON(t, customClusterJSONOptions{ + Name: "myservice", + }) + }, nil) }, }, { - name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", + name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, }, + } +} + +func TestEndpointsFromSnapshot(t *testing.T) { + // TODO: we should move all of these to TestAllResourcesFromSnapshot + // eventually to test all of the xDS types at once with the same input, + // just as it would be triggered by our xDS server. + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + tests := []endpointTestCase{ { - name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", + name: "mesh-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", nil, nil) + return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", + name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", nil, nil) + return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) }, }, { - name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", + name: "mesh-gateway-newer-information-in-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", nil, nil) + return proxycfg.TestConfigSnapshotMeshGateway(t, "newer-info-in-federation-states", nil, nil) }, }, { - name: "connect-proxy-with-default-chain-and-custom-cluster", + name: "mesh-gateway-older-information-in-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", func(ns *structs.NodeService) { - ns.Proxy.Upstreams[0].Config["envoy_cluster_json"] = - customAppClusterJSON(t, customClusterJSONOptions{ - Name: "myservice", - }) - }, nil) + return proxycfg.TestConfigSnapshotMeshGateway(t, "older-info-in-federation-states", nil, nil) }, }, { - name: "splitter-with-resolver-redirect", + name: "mesh-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", nil, nil) + return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) }, }, { @@ -498,6 +505,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { }, } + tests = append(tests, makeEndpointDiscoChainTests(false)...) + latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { sf, err := xdscommon.DetermineSupportedProxyFeaturesFromString(envoyVersion) diff --git a/agent/xds/extensionruntime/runtime_config_oss_test.go b/agent/xds/extensionruntime/runtime_config_oss_test.go index 62ce5f812c18..cc004c788de8 100644 --- a/agent/xds/extensionruntime/runtime_config_oss_test.go +++ b/agent/xds/extensionruntime/runtime_config_oss_test.go @@ -130,11 +130,11 @@ func TestGetRuntimeConfigurations_ConnectProxy(t *testing.T) { } // Setup a snapshot where the db upstream is on a connect proxy. - snapConnect := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, serviceDefaults) + snapConnect := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, serviceDefaults) // Setup a snapshot where the db upstream is on a terminating gateway. - snapTermGw := proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", nil, nil, serviceDefaults) + snapTermGw := proxycfg.TestConfigSnapshotDiscoveryChain(t, "register-to-terminating-gateway", false, nil, nil, serviceDefaults) // Setup a snapshot with the local service web has extensions. - snapWebConnect := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", func(ns *structs.NodeService) { + snapWebConnect := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, func(ns *structs.NodeService) { ns.Proxy.EnvoyExtensions = envoyExtensions }, nil) diff --git a/agent/xds/golden_test.go b/agent/xds/golden_test.go index 420285203e03..c056aa7b3a04 100644 --- a/agent/xds/golden_test.go +++ b/agent/xds/golden_test.go @@ -1,6 +1,7 @@ package xds import ( + "encoding/json" "flag" "fmt" "os" @@ -8,6 +9,7 @@ import ( "testing" "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -83,12 +85,16 @@ func golden(t *testing.T, name, subname, latestSubname, got string) string { golden := filepath.Join("testdata", name+suffix) - // Always load the latest golden file if configured to do so. - latestExpected := "" - if latestSubname != "" && subname != latestSubname { - latestGolden := filepath.Join("testdata", fmt.Sprintf("%s.%s.golden", name, latestSubname)) - raw, err := os.ReadFile(latestGolden) - require.NoError(t, err, "%q %q %q", name, subname, latestSubname) + var latestGoldenPath, latestExpected string + isLatest := subname == latestSubname + // Include latestSubname in the latest golden path if it exists. + if latestSubname == "" { + latestGoldenPath = filepath.Join("testdata", fmt.Sprintf("%s.golden", name)) + } else { + latestGoldenPath = filepath.Join("testdata", fmt.Sprintf("%s.%s.golden", name, latestSubname)) + } + + if raw, err := os.ReadFile(latestGoldenPath); err == nil { latestExpected = string(raw) } @@ -97,8 +103,14 @@ func golden(t *testing.T, name, subname, latestSubname, got string) string { // // To trim down PRs, we only create per-version golden files if they differ // from the latest version. + if *update && got != "" { - if latestExpected == got { + var gotInterface, latestExpectedInterface interface{} + json.Unmarshal([]byte(got), &gotInterface) + json.Unmarshal([]byte(latestExpected), &latestExpectedInterface) + + // Remove non-latest golden files if they are the same as the latest one. + if !isLatest && assert.ObjectsAreEqualValues(gotInterface, latestExpectedInterface) { // In update mode we erase a golden file if it is identical to // the golden file corresponding to the latest version of // envoy. @@ -109,7 +121,12 @@ func golden(t *testing.T, name, subname, latestSubname, got string) string { return got } - require.NoError(t, os.WriteFile(golden, []byte(got), 0644)) + // We use require.JSONEq to compare values and ObjectsAreEqualValues is used + // internally by that function to compare the string JSON values. This only + // writes updates the golden file when they actually change. + if !assert.ObjectsAreEqualValues(gotInterface, latestExpectedInterface) { + require.NoError(t, os.WriteFile(golden, []byte(got), 0644)) + } return got } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 6d0d64407bbf..a69a74a2b1c5 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -21,6 +21,121 @@ import ( "github.com/hashicorp/consul/types" ) +type listenerTestCase struct { + name string + create func(t testinf.T) *proxycfg.ConfigSnapshot + // Setup is called before the test starts. It is passed the snapshot from + // TestConfigSnapshot and is allowed to modify it in any way to setup the + // test input. + overrideGoldenName string + generatorSetup func(*ResourceGenerator) +} + +func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { + return []listenerTestCase{ + { + name: "custom-upstream-ignored-with-disco-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, func(ns *structs.NodeService) { + for i := range ns.Proxy.Upstreams { + if ns.Proxy.Upstreams[i].DestinationName != "db" { + continue // only tweak the db upstream + } + if ns.Proxy.Upstreams[i].Config == nil { + ns.Proxy.Upstreams[i].Config = map[string]interface{}{} + } + + uid := proxycfg.NewUpstreamID(&ns.Proxy.Upstreams[i]) + + ns.Proxy.Upstreams[i].Config["envoy_listener_json"] = + customListenerJSON(t, customListenerJSONOptions{ + Name: uid.EnvoyID() + ":custom-upstream", + }) + } + }, nil) + }, + }, + { + name: "splitter-with-resolver-redirect", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-http-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "http", + }, + }, + ) + }, + }, + { + name: "connect-proxy-with-http2-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "http2", + }, + }, + ) + }, + }, + { + name: "connect-proxy-with-grpc-chain", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "grpc", + }, + }, + ) + }, + }, + { + name: "connect-proxy-with-chain-external-sni", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-chain-and-overrides", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) + }, + }, + { + name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) + }, + }, + } +} + func TestListenersFromSnapshot(t *testing.T) { // TODO: we should move all of these to TestAllResourcesFromSnapshot // eventually to test all of the xDS types at once with the same input, @@ -29,16 +144,7 @@ func TestListenersFromSnapshot(t *testing.T) { t.Skip("too slow for testing.Short") } - tests := []struct { - name string - create func(t testinf.T) *proxycfg.ConfigSnapshot - // Setup is called before the test starts. It is passed the snapshot from - // TestConfigSnapshot and is allowed to modify it in any way to setup the - // test input. - setup func(snap *proxycfg.ConfigSnapshot) - overrideGoldenName string - generatorSetup func(*ResourceGenerator) - }{ + tests := []listenerTestCase{ { name: "connect-proxy-with-tls-outgoing-min-version-auto", create: func(t testinf.T) *proxycfg.ConfigSnapshot { @@ -326,106 +432,6 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, }, - { - name: "custom-upstream-ignored-with-disco-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", func(ns *structs.NodeService) { - for i := range ns.Proxy.Upstreams { - if ns.Proxy.Upstreams[i].DestinationName != "db" { - continue // only tweak the db upstream - } - if ns.Proxy.Upstreams[i].Config == nil { - ns.Proxy.Upstreams[i].Config = map[string]interface{}{} - } - - uid := proxycfg.NewUpstreamID(&ns.Proxy.Upstreams[i]) - - ns.Proxy.Upstreams[i].Config["envoy_listener_json"] = - customListenerJSON(t, customListenerJSONOptions{ - Name: uid.EnvoyID() + ":custom-upstream", - }) - } - }, nil) - }, - }, - { - name: "splitter-with-resolver-redirect", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil) - }, - }, - { - name: "connect-proxy-with-http-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil, - &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, - Config: map[string]interface{}{ - "protocol": "http", - }, - }, - ) - }, - }, - { - name: "connect-proxy-with-http2-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil, - &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, - Config: map[string]interface{}{ - "protocol": "http2", - }, - }, - ) - }, - }, - { - name: "connect-proxy-with-grpc-chain", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil, - &structs.ProxyConfigEntry{ - Kind: structs.ProxyDefaults, - Name: structs.ProxyConfigGlobal, - Config: map[string]interface{}{ - "protocol": "grpc", - }, - }, - ) - }, - }, - { - name: "connect-proxy-with-chain-external-sni", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", nil, nil) - }, - }, - { - name: "connect-proxy-with-chain-and-overrides", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", nil, nil) - }, - }, - { - name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", nil, nil) - }, - }, { name: "connect-proxy-upstream-defaults", create: func(t testinf.T) *proxycfg.ConfigSnapshot { @@ -1092,6 +1098,8 @@ func TestListenersFromSnapshot(t *testing.T) { }, } + tests = append(tests, makeListenerDiscoChainTests(false)...) + latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { sf, err := xdscommon.DetermineSupportedProxyFeaturesFromString(envoyVersion) @@ -1110,10 +1118,6 @@ func TestListenersFromSnapshot(t *testing.T) { // golder files for every test case and so not be any use! testcommon.SetupTLSRootsAndLeaf(t, snap) - if tt.setup != nil { - tt.setup(snap) - } - // Need server just for logger dependency g := NewResourceGenerator(testutil.Logger(t), nil, false) g.ProxyFeatures = sf diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 24bee7660615..1f9b99274bdd 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -175,7 +175,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) - tests = append(tests, getTrafficControlPeeringGoldenTestCases()...) + tests = append(tests, getTrafficControlPeeringGoldenTestCases(false)...) tests = append(tests, getEnterpriseGoldenTestCases()...) tests = append(tests, getAPIGatewayGoldenTestCases(t)...) @@ -253,21 +253,29 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { } } -func getTrafficControlPeeringGoldenTestCases() []goldenTestCase { - return []goldenTestCase{ +func getTrafficControlPeeringGoldenTestCases(enterprise bool) []goldenTestCase { + cases := []goldenTestCase{ { name: "connect-proxy-with-chain-and-failover-to-cluster-peer", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", enterprise, nil, nil) }, }, { name: "connect-proxy-with-chain-and-redirect-to-cluster-peer", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", enterprise, nil, nil) }, }, } + + if enterprise { + for i := range cases { + cases[i].name = "enterprise-" + cases[i].name + } + } + + return cases } const ( diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index cec0f650c7a5..c68e54c6d0a7 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -19,67 +19,74 @@ import ( "github.com/hashicorp/consul/sdk/testutil" ) -func TestRoutesFromSnapshot(t *testing.T) { - // TODO: we should move all of these to TestAllResourcesFromSnapshot - // eventually to test all of the xDS types at once with the same input, - // just as it would be triggered by our xDS server. - if testing.Short() { - t.Skip("too slow for testing.Short") - } +type routeTestCase struct { + name string + create func(t testinf.T) *proxycfg.ConfigSnapshot + overrideGoldenName string +} - tests := []struct { - name string - create func(t testinf.T) *proxycfg.ConfigSnapshot - overrideGoldenName string - }{ +func makeRouteDiscoChainTests(enterprise bool) []routeTestCase { + return []routeTestCase{ { name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, }, { name: "connect-proxy-with-chain-and-splitter", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-splitter", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-splitter", enterprise, nil, nil) }, }, { name: "connect-proxy-with-grpc-router", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "grpc-router", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "grpc-router", enterprise, nil, nil) }, }, { name: "connect-proxy-with-chain-and-router", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-router", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-router", enterprise, nil, nil) }, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, }, + } +} + +func TestRoutesFromSnapshot(t *testing.T) { + // TODO: we should move all of these to TestAllResourcesFromSnapshot + // eventually to test all of the xDS types at once with the same input, + // just as it would be triggered by our xDS server. + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + tests := []routeTestCase{ // TODO(rb): test match stanza skipped for grpc // Start ingress gateway test cases { @@ -188,6 +195,8 @@ func TestRoutesFromSnapshot(t *testing.T) { }, } + tests = append(tests, makeRouteDiscoChainTests(false)...) + latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { sf, err := xdscommon.DetermineSupportedProxyFeaturesFromString(envoyVersion) diff --git a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index 8cb6ce20a06d..e55cdc39f7a1 100644 --- a/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/endpoints/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -1,63 +1,63 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "clusterName": "db.default.cluster-01.external.peer1.domain", - "endpoints": [ + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.cluster-01.external.peer1.domain", + "endpoints": [ { - "lbEndpoints": [ + "lbEndpoints": [ { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.40.1.1", - "portValue": 8080 + "endpoint": { + "address": { + "socketAddress": { + "address": "10.40.1.1", + "portValue": 8080 } } }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 } ] } ] }, { - "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", - "endpoints": [ + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ { - "lbEndpoints": [ + "lbEndpoints": [ { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.10.1.1", - "portValue": 8080 + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 } } }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 }, { - "endpoint": { - "address": { - "socketAddress": { - "address": "10.20.1.2", - "portValue": 8080 + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 } } }, - "healthStatus": "HEALTHY", - "loadBalancingWeight": 1 + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 } ] } ] } ], - "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden index 57d50f71c367..5fdd2e351c0e 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -1,119 +1,115 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "db:127.0.0.1:9191", - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 9191 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.db.default.default.dc1", - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "prepared_query:geo-cache:127.10.10.10:8181", - "address": { - "socketAddress": { - "address": "127.10.10.10", - "portValue": 8181 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.prepared_query_geo-cache", - "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "public_listener:0.0.0.0:9999", - "address": { - "socketAddress": { - "address": "0.0.0.0", - "portValue": 9999 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.rbac", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", - "rules": { - - }, - "statPrefix": "connect_authz" + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": {}, + "statPrefix": "connect_authz" } }, { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "public_listener", - "cluster": "local_app" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" } } ], - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" } } }, - "requireClientCertificate": true + "requireClientCertificate": true } } } ], - "trafficDirection": "INBOUND" + "trafficDirection": "INBOUND" } ], - "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index e061148c0088..02d749a2c5c2 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -1,119 +1,115 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "db:127.0.0.1:9191", - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 9191 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.db.default.default.dc1", - "cluster": "db.default.cluster-01.external.peer1.domain" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.cluster-01.external.peer1.domain" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "prepared_query:geo-cache:127.10.10.10:8181", - "address": { - "socketAddress": { - "address": "127.10.10.10", - "portValue": 8181 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.prepared_query_geo-cache", - "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" } } ] } ], - "trafficDirection": "OUTBOUND" + "trafficDirection": "OUTBOUND" }, { - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", - "name": "public_listener:0.0.0.0:9999", - "address": { - "socketAddress": { - "address": "0.0.0.0", - "portValue": 9999 + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 } }, - "filterChains": [ + "filterChains": [ { - "filters": [ + "filters": [ { - "name": "envoy.filters.network.rbac", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", - "rules": { - - }, - "statPrefix": "connect_authz" + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": {}, + "statPrefix": "connect_authz" } }, { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "public_listener", - "cluster": "local_app" + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" } } ], - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" } } }, - "requireClientCertificate": true + "requireClientCertificate": true } } } ], - "trafficDirection": "INBOUND" + "trafficDirection": "INBOUND" } ], - "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/validateupstream-test/validateupstream_test.go b/agent/xds/validateupstream-test/validateupstream_test.go index 6b758f853831..c85fe49b0f00 100644 --- a/agent/xds/validateupstream-test/validateupstream_test.go +++ b/agent/xds/validateupstream-test/validateupstream_test.go @@ -43,13 +43,13 @@ func TestValidateUpstreams(t *testing.T) { { name: "tcp-success", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil) }, }, { name: "tcp-missing-listener", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil) }, patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources { delete(ir.Index[xdscommon.ListenerType], listenerName) @@ -60,7 +60,7 @@ func TestValidateUpstreams(t *testing.T) { { name: "tcp-missing-cluster", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil) }, patcher: func(ir *xdscommon.IndexedResources) *xdscommon.IndexedResources { delete(ir.Index[xdscommon.ClusterType], sni) @@ -71,7 +71,7 @@ func TestValidateUpstreams(t *testing.T) { { name: "http-success", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, httpServiceDefaults) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, httpServiceDefaults) }, }, { @@ -79,7 +79,7 @@ func TestValidateUpstreams(t *testing.T) { // RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain, so we // need to use the test snapshot and add L7 config entries. create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, []proxycfg.UpdateEvent{ + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, []proxycfg.UpdateEvent{ // The events ensure there are endpoints for the v1 and v2 subsets. { CorrelationID: "upstream-target:v1.db.default.default.dc1:" + dbUID.String(), @@ -104,7 +104,7 @@ func TestValidateUpstreams(t *testing.T) { // RDS, Envoy's Route Discovery Service, is only used for HTTP services with a customized discovery chain, so we // need to use the test snapshot and add L7 config entries. create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, []proxycfg.UpdateEvent{ + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, []proxycfg.UpdateEvent{ // The events ensure there are endpoints for the v1 and v2 subsets. { CorrelationID: "upstream-target:v1.db.default.default.dc1:" + dbUID.String(), @@ -129,19 +129,19 @@ func TestValidateUpstreams(t *testing.T) { { name: "redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", false, nil, nil) }, }, { name: "failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", false, nil, nil) }, }, { name: "failover-to-cluster-peer", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", nil, nil) + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", false, nil, nil) }, }, { diff --git a/agent/xds/xds_protocol_helpers_test.go b/agent/xds/xds_protocol_helpers_test.go index 2edd05b9fb20..b71c8282e8f1 100644 --- a/agent/xds/xds_protocol_helpers_test.go +++ b/agent/xds/xds_protocol_helpers_test.go @@ -47,7 +47,7 @@ func newTestSnapshot( dbServiceProtocol string, additionalEntries ...structs.ConfigEntry, ) *proxycfg.ConfigSnapshot { - snap := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", nil, nil, additionalEntries...) + snap := proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nil, nil, additionalEntries...) snap.ConnectProxy.PreparedQueryEndpoints = map[proxycfg.UpstreamID]structs.CheckServiceNodes{ UID("prepared_query:geo-cache"): proxycfg.TestPreparedQueryNodes(t, "geo-cache"), } diff --git a/command/services/config_test.go b/command/services/config_test.go index 1647b9293425..b54a793aa474 100644 --- a/command/services/config_test.go +++ b/command/services/config_test.go @@ -137,7 +137,7 @@ func TestStructsToAgentService(t *testing.T) { DestinationServiceName: "web", LocalServiceAddress: "127.0.0.1", LocalServicePort: 8181, - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), Mode: structs.ProxyModeTransparent, Config: map[string]interface{}{ "foo": "bar", @@ -154,7 +154,7 @@ func TestStructsToAgentService(t *testing.T) { DestinationServiceName: "web", LocalServiceAddress: "127.0.0.1", LocalServicePort: 8181, - Upstreams: structs.TestUpstreams(t).ToAPI(), + Upstreams: structs.TestUpstreams(t, false).ToAPI(), Mode: api.ProxyModeTransparent, Config: map[string]interface{}{ "foo": "bar", @@ -174,7 +174,7 @@ func TestStructsToAgentService(t *testing.T) { DestinationServiceName: "web", LocalServiceAddress: "127.0.0.1", LocalServicePort: 8181, - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), Mode: structs.ProxyModeTransparent, TransparentProxy: structs.TransparentProxyConfig{ OutboundListenerPort: 808, @@ -201,7 +201,7 @@ func TestStructsToAgentService(t *testing.T) { DestinationServiceName: "web", LocalServiceAddress: "127.0.0.1", LocalServicePort: 8181, - Upstreams: structs.TestUpstreams(t).ToAPI(), + Upstreams: structs.TestUpstreams(t, false).ToAPI(), Mode: api.ProxyModeTransparent, TransparentProxy: &api.TransparentProxyConfig{ OutboundListenerPort: 808, From d06ede9daf9b2420cddb18ce69cfbafbeccdeb35 Mon Sep 17 00:00:00 2001 From: Dan Bond Date: Thu, 23 Mar 2023 16:40:48 -0700 Subject: [PATCH 125/421] ci: use correct build-distros workflow (#16757) Signed-off-by: Dan Bond --- .github/workflows/build-distros.yml | 60 ++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 9a43a7f52065..87d6f9605442 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -1,17 +1,32 @@ +# NOTE: this workflow builds Consul binaries on multiple architectures for PRs. +# It is aimed at checking new commits don't introduce any breaking build changes. name: build-distros -on: - pull_request: - branches: ["main"] - push: - branches: ["main"] - tags: ["*"] +on: [pull_request] + +permissions: + contents: read jobs: + check-go-mod: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go mod tidy + - run: | + if [[ -n $(git status -s) ]]; then + echo "Git directory has changes" + git status -s + exit 1 + fi + build-386: - strategy: - matrix: - os: ['freebsd', 'linux', 'windows'] + needs: check-go-mod + env: + XC_OS: "freebsd linux windows" runs-on: ubuntu-22.04 steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -19,12 +34,15 @@ jobs: with: go-version-file: 'go.mod' - name: Build - run: GOOS=${{ matrix.os }} GOARCH=386 CGO_ENABLED=0 go build + run: | + for os in $XC_OS; do + GOOS="$os" GOARCH=386 CGO_ENABLED=0 go build + done build-amd64: - strategy: - matrix: - os: ['darwin', 'freebsd', 'linux', 'solaris', 'windows'] + needs: check-go-mod + env: + XC_OS: "darwin freebsd linux solaris windows" runs-on: ubuntu-22.04 steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -32,12 +50,26 @@ jobs: with: go-version-file: 'go.mod' - name: Build - run: GOOS=${{ matrix.os }} GOARCH=amd64 CGO_ENABLED=0 go build + run: | + for os in $XC_OS; do + GOOS="$os" GOARCH=amd64 CGO_ENABLED=0 go build + done build-arm: + needs: check-go-mod runs-on: ubuntu-22.04 + env: + CGO_ENABLED: 1 + GOOS: linux steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' + - run: | + sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list # workaround for https://github.com/actions/runner-images/issues/1983 + sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu + + - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build + - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build + - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build From 52c51bd9459d3abcba108a99931f297461192883 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 24 Mar 2023 09:23:46 -0700 Subject: [PATCH 126/421] Backport of Changelog for audit logging fix. into release/1.15.x (#16744) * backport of commit e72e2bb2f73c2e8e075fd394d43057b715519afd * backport of commit f4cdcb208ebc0af2d8a6eedc1aeeca3982d01843 * backport of commit e720a4a8c1aa544b9b026ccc02bc0fdbeb7fe921 --------- Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --- .changelog/16700.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/16700.txt diff --git a/.changelog/16700.txt b/.changelog/16700.txt new file mode 100644 index 000000000000..82da5936ddb5 --- /dev/null +++ b/.changelog/16700.txt @@ -0,0 +1,3 @@ +```release-note:bug +audit-logging: (Enterprise only) Fix a bug where `/agent/monitor` and `/agent/metrics` endpoints return a `Streaming not supported` error when audit logs are enabled. This also fixes the delay receiving logs when running `consul monitor` against an agent with audit logs enabled. +``` From 591850d2368541f64d73dd80011fc50c7842d2f4 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 24 Mar 2023 11:23:05 -0700 Subject: [PATCH 127/421] Backport of RELENG-471: Remove obsolete load-test workflow into release/1.15.x (#16765) * no-op commit due to failed cherry-picking * RELENG-471: Remove obsolete load-test workflow (#16737) * Remove obsolete load-test workflow * remove load-tests from circleci config. --------- Co-authored-by: John Murret * RELENG-471: Remove obsolete load-test workflow (#16737) * Remove obsolete load-test workflow * remove load-tests from circleci config. --------- Co-authored-by: John Murret --------- Co-authored-by: temp Co-authored-by: brian shore Co-authored-by: John Murret --- .circleci/config.yml | 96 --------------------------------- .github/workflows/load-test.yml | 66 ----------------------- 2 files changed, 162 deletions(-) delete mode 100644 .github/workflows/load-test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 46ebfe240e91..20bcf317403e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,10 +6,6 @@ parameters: type: string default: "" description: "Commit to run load tests against" - trigger-load-test: - type: boolean - default: false - description: "Boolean whether to run the load test workflow" references: paths: @@ -992,78 +988,6 @@ jobs: path: *TEST_RESULTS_DIR - run: *notify-slack-failure - # Run load tests against a commit - load-test: - docker: - - image: hashicorp/terraform:latest - environment: - AWS_DEFAULT_REGION: us-east-2 - BUCKET: consul-ci-load-tests - BASH_ENV: /etc/profile - shell: /bin/sh -leo pipefail - steps: - - checkout - - run: apk add jq curl bash - - run: - name: export load-test credentials - command: | - echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_LOAD_TEST" >> $BASH_ENV - echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_LOAD_TEST" >> $BASH_ENV - - run: - name: export role arn - command: | - echo "export TF_VAR_role_arn=$ROLE_ARN_LOAD_TEST" >> $BASH_ENV - - run: - name: setup TF_VARs - command: | - # if pipeline.parameters.commit="" it was not triggered/set through the API - # so we use the latest commit from _this_ branch. This is the case for nightly tests. - if [ "<< pipeline.parameters.commit >>" = "" ]; then - LOCAL_COMMIT_SHA=$(git rev-parse HEAD) - else - LOCAL_COMMIT_SHA="<< pipeline.parameters.commit >>" - fi - echo "export LOCAL_COMMIT_SHA=${LOCAL_COMMIT_SHA}" >> $BASH_ENV - git checkout ${LOCAL_COMMIT_SHA} - - short_ref=$(git rev-parse --short ${LOCAL_COMMIT_SHA}) - echo "export TF_VAR_ami_owners=$LOAD_TEST_AMI_OWNERS" >> $BASH_ENV - echo "export TF_VAR_vpc_name=$short_ref" >> $BASH_ENV - echo "export TF_VAR_cluster_name=$short_ref" >> $BASH_ENV - echo "export TF_VAR_consul_download_url=https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" >> $BASH_ENV - - run: - name: wait for dev build from test-integrations workflow - command: | - echo "curl-ing https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" - until [ $SECONDS -ge 300 ] && exit 1; do - curl -o /dev/null --fail --silent "https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" && exit - echo -n "." - sleep 2 - done - - run: - working_directory: .circleci/terraform/load-test - name: terraform init - command: | - short_ref=$(git rev-parse --short HEAD) - echo "Testing commit id: $short_ref" - terraform init \ - -backend-config="bucket=${BUCKET}" \ - -backend-config="key=${LOCAL_COMMIT_SHA}" \ - -backend-config="region=${AWS_DEFAULT_REGION}" \ - -backend-config="role_arn=${ROLE_ARN_LOAD_TEST}" - - run: - working_directory: .circleci/terraform/load-test - name: run terraform apply - command: | - terraform apply -auto-approve - - run: - working_directory: .circleci/terraform/load-test - when: always - name: terraform destroy - command: | - for i in $(seq 1 5); do terraform destroy -auto-approve && s=0 && break || s=$? && sleep 20; done; (exit $s) - - run: *notify-slack-failure - # The noop job is a used as a very fast job in the verify-ci workflow because every workflow # requires at least one job. It does nothing. noop: @@ -1080,7 +1004,6 @@ workflows: jobs: [noop] go-tests: - unless: << pipeline.parameters.trigger-load-test >> jobs: - check-go-mod: &filter-ignore-non-go-branches filters: @@ -1143,7 +1066,6 @@ workflows: - go-test-32bit: *filter-ignore-non-go-branches - noop build-distros: - unless: << pipeline.parameters.trigger-load-test >> jobs: - check-go-mod: *filter-ignore-non-go-branches - build-386: &require-check-go-mod @@ -1175,7 +1097,6 @@ workflows: context: consul-ci - noop test-integrations: - unless: << pipeline.parameters.trigger-load-test >> jobs: - dev-build: *filter-ignore-non-go-branches - dev-upload-s3: &dev-upload @@ -1212,7 +1133,6 @@ workflows: - dev-build - noop frontend: - unless: << pipeline.parameters.trigger-load-test >> jobs: - frontend-cache: filters: @@ -1245,19 +1165,3 @@ workflows: requires: - ember-build-ent - noop - - load-test: - when: << pipeline.parameters.trigger-load-test >> - jobs: - - load-test - - nightly-jobs: - triggers: - - schedule: - cron: "0 4 * * *" # 4AM UTC <> 12AM EST <> 9PM PST should have no impact - filters: - branches: - only: - - main - jobs: - - load-test diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml deleted file mode 100644 index ab7d793e7945..000000000000 --- a/.github/workflows/load-test.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Load Test - -on: - pull_request: - branches: - - main - types: [labeled] - workflow_dispatch: {} - -jobs: - trigger-load-test: - if: ${{ github.event.label.name == 'pr/load-test' }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Trigger CircleCI Load Test Pipeline - run: | - # build json payload to trigger CircleCI Load Test Pipeline - # This only runs the load test pipeline on the 'main' branch - jsonData=$(jq --null-input -r --arg commit ${{ github.event.pull_request.head.sha }} \ - '{branch:"main", parameters: {"commit": $commit, "trigger-load-test": true}}') - echo "Passing JSON data to CircleCI API: $jsonData" - load_test_pipeline_id=$(curl -X POST \ - -H "Circle-Token: ${{ secrets.CIRCLE_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d "$jsonData" \ - "https://circleci.com/api/v2/project/gh/${GITHUB_REPOSITORY}/pipeline" | jq -r '.id') - echo "LOAD_TEST_PIPELINE_ID=$load_test_pipeline_id" >> $GITHUB_ENV - - name: Post Load Test URL to PR - env: - PR_COMMENT_URL: ${{ github.event.pull_request.comments_url }} - run: | - echo "LOAD_TEST_PIPELINE_ID is: $LOAD_TEST_PIPELINE_ID" - # get load-test workflow - workflow= - # wait up to a minute for load-test workflow to start - until [ $SECONDS -ge 60 ] && exit 1; do - workflow=$(curl -s -X GET \ - -H "Circle-Token: ${{ secrets.CIRCLE_TOKEN }}" \ - -H "Content-Type: application/json" \ - "https://circleci.com/api/v2/pipeline/${LOAD_TEST_PIPELINE_ID}/workflow" | jq '.items[] | select(.name=="load-test")') - # if we found a workflow we exit - if [ -n "$workflow" ]; then - break - fi - echo -n "." - sleep 5 - done - echo "$workflow" - # get pipeline number - pipeline_number=$(echo "$workflow" | jq -r '.pipeline_number') - # get workflow id - workflow_id=$(echo "$workflow" | jq -r '.id') - # get project slug - project_slug=$(echo "$workflow" | jq -r '.project_slug') - # build load test URL - load_test_url="https://app.circleci.com/pipelines/${project_slug}/${pipeline_number}/workflows/${workflow_id}" - # comment URL to pull request - curl -X POST \ - -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d "{\"body\": \"Load Test Pipeline Started at: $load_test_url\"}" \ - "$PR_COMMENT_URL" From dfb7f7138d87354fd9dea98ae3d24303e6aef561 Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Fri, 24 Mar 2023 15:40:46 -0500 Subject: [PATCH 128/421] Change partition for peers in discovery chain targets (#16770) This commit swaps the partition field to the local partition for discovery chains targeting peers. Prior to this change, peer upstreams would always use a value of default regardless of which partition they exist in. This caused several issues in xds / proxycfg because of id mismatches. Some prior fixes were made to deal with one-off id mismatches that this PR also cleans up, since they are no longer needed. --- .changelog/_4832.txt | 3 + agent/consul/discoverychain/compile.go | 17 +- agent/proxycfg/testing_api_gateway.go | 2 +- agent/proxycfg/testing_connect_proxy.go | 2 +- agent/proxycfg/testing_ingress_gateway.go | 2 +- agent/proxycfg/testing_mesh_gateway.go | 11 +- agent/proxycfg/testing_upstreams.go | 47 +++- agent/proxycfg/upstreams.go | 24 +- agent/structs/discovery_chain.go | 2 +- agent/structs/testing_catalog.go | 19 +- agent/xds/clusters.go | 4 - ...and-failover-to-cluster-peer.latest.golden | 256 ++++++++---------- ...and-redirect-to-cluster-peer.latest.golden | 166 +++++------- ...and-failover-to-cluster-peer.latest.golden | 170 +++++------- 14 files changed, 341 insertions(+), 384 deletions(-) create mode 100644 .changelog/_4832.txt diff --git a/.changelog/_4832.txt b/.changelog/_4832.txt new file mode 100644 index 000000000000..b45768715540 --- /dev/null +++ b/.changelog/_4832.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: **(Consul Enterprise only)** Fix issue where resolvers, routers, and splitters referencing peer targets may not work correctly for non-default partitions and namespaces. Enterprise customers leveraging peering are encouraged to upgrade both servers and agents to avoid this problem. +``` diff --git a/agent/consul/discoverychain/compile.go b/agent/consul/discoverychain/compile.go index ead109d419e4..212c3d8fbd3a 100644 --- a/agent/consul/discoverychain/compile.go +++ b/agent/consul/discoverychain/compile.go @@ -719,10 +719,21 @@ func (c *compiler) newTarget(opts structs.DiscoveryTargetOpts) *structs.Discover } else { // Don't allow Peer and Datacenter. opts.Datacenter = "" - // Peer and Partition cannot both be set. - opts.Partition = acl.PartitionOrDefault("") + // Since discovery targets (for peering) are ONLY used to query the catalog, and + // not to generate the SNI it is more correct to switch this to the calling-side + // of the peering's partition as that matches where the replicated data is stored + // in the catalog. This is done to simplify the usage of peer targets in both + // the xds and proxycfg packages. + // + // The peer info data attached to service instances will have the embedded opaque + // SNI/SAN information generated by the remote side and that will have the + // OTHER partition properly specified. + opts.Partition = acl.PartitionOrDefault(c.evaluateInPartition) // Default to "default" rather than c.evaluateInNamespace. - opts.Namespace = acl.PartitionOrDefault(opts.Namespace) + // Note that the namespace is not swapped out, because it should + // always match the value in the remote cluster (and shouldn't + // have been changed anywhere). + opts.Namespace = acl.NamespaceOrDefault(opts.Namespace) } t := structs.NewDiscoveryTarget(opts) diff --git a/agent/proxycfg/testing_api_gateway.go b/agent/proxycfg/testing_api_gateway.go index b63c624e05a9..80a50e2cebaa 100644 --- a/agent/proxycfg/testing_api_gateway.go +++ b/agent/proxycfg/testing_api_gateway.go @@ -108,7 +108,7 @@ func TestConfigSnapshotAPIGateway( upstreams := structs.TestUpstreams(t, false) baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot( - t, variation, upstreams, additionalEntries..., + t, variation, false, upstreams, additionalEntries..., )) return testConfigSnapshotFixture(t, &structs.NodeService{ diff --git a/agent/proxycfg/testing_connect_proxy.go b/agent/proxycfg/testing_connect_proxy.go index 332c8ef2d8f3..5624a36e647b 100644 --- a/agent/proxycfg/testing_connect_proxy.go +++ b/agent/proxycfg/testing_connect_proxy.go @@ -145,7 +145,7 @@ func TestConfigSnapshotDiscoveryChain( }, }, }, setupTestVariationConfigEntriesAndSnapshot( - t, variation, upstreams, additionalEntries..., + t, variation, enterprise, upstreams, additionalEntries..., )) return testConfigSnapshotFixture(t, &structs.NodeService{ diff --git a/agent/proxycfg/testing_ingress_gateway.go b/agent/proxycfg/testing_ingress_gateway.go index 274f9931b75e..5e981b9e579e 100644 --- a/agent/proxycfg/testing_ingress_gateway.go +++ b/agent/proxycfg/testing_ingress_gateway.go @@ -88,7 +88,7 @@ func TestConfigSnapshotIngressGateway( upstreams = structs.Upstreams{upstreams[0]} // just keep 'db' baseEvents = testSpliceEvents(baseEvents, setupTestVariationConfigEntriesAndSnapshot( - t, variation, upstreams, additionalEntries..., + t, variation, false, upstreams, additionalEntries..., )) } diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go index f78452f26929..87d4f33082aa 100644 --- a/agent/proxycfg/testing_mesh_gateway.go +++ b/agent/proxycfg/testing_mesh_gateway.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" @@ -474,6 +475,8 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( discoChains = make(map[structs.ServiceName]*structs.CompiledDiscoveryChain) endpoints = make(map[structs.ServiceName]structs.CheckServiceNodes) entries []structs.ConfigEntry + // This portion of the test is not currently enterprise-aware, but we need this to satisfy a function call. + entMeta = *acl.DefaultEnterpriseMeta() ) switch variant { @@ -660,8 +663,8 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( CorrelationID: "peering-connect-service:peer-a:db", Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{ - structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.1", false), - structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.2", false), + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.1", false, entMeta), + structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "peer-a", "10.40.1.2", false, entMeta), }, }, }, @@ -669,8 +672,8 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( CorrelationID: "peering-connect-service:peer-b:alt", Result: &structs.IndexedCheckServiceNodes{ Nodes: structs.CheckServiceNodes{ - structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.1", false), - structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.2", true), + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.1", false, entMeta), + structs.TestCheckNodeServiceWithNameInPeer(t, "alt", "remote-dc", "peer-b", "10.40.2.2", true, entMeta), }, }, }, diff --git a/agent/proxycfg/testing_upstreams.go b/agent/proxycfg/testing_upstreams.go index 0e09aff9e104..8077bbb12ebe 100644 --- a/agent/proxycfg/testing_upstreams.go +++ b/agent/proxycfg/testing_upstreams.go @@ -15,6 +15,7 @@ import ( func setupTestVariationConfigEntriesAndSnapshot( t testing.T, variation string, + enterprise bool, upstreams structs.Upstreams, additionalEntries ...structs.ConfigEntry, ) []UpdateEvent { @@ -24,7 +25,7 @@ func setupTestVariationConfigEntriesAndSnapshot( dbUID = NewUpstreamID(&dbUpstream) ) - dbChain := setupTestVariationDiscoveryChain(t, variation, dbUID.EnterpriseMeta, additionalEntries...) + dbChain := setupTestVariationDiscoveryChain(t, variation, enterprise, dbUID.EnterpriseMeta, additionalEntries...) nodes := TestUpstreamNodes(t, "db") if variation == "register-to-terminating-gateway" { @@ -106,14 +107,16 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }) uid := UpstreamID{ - Name: "db", - Peer: "cluster-01", - EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), ""), + Name: "db", + Peer: "cluster-01", + } + if enterprise { + uid.EnterpriseMeta = acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), "ns9") } events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:" + uid.String(), Result: &structs.IndexedCheckServiceNodes{ - Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc1", "cluster-01", "10.40.1.1", false)}, + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc2", "cluster-01", "10.40.1.1", false, uid.EnterpriseMeta)}, }, }) case "redirect-to-cluster-peer": @@ -129,14 +132,16 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }) uid := UpstreamID{ - Name: "db", - Peer: "cluster-01", - EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), ""), + Name: "db", + Peer: "cluster-01", + } + if enterprise { + uid.EnterpriseMeta = acl.NewEnterpriseMetaWithPartition(dbUID.PartitionOrDefault(), "ns9") } events = append(events, UpdateEvent{ CorrelationID: "upstream-peer:" + uid.String(), Result: &structs.IndexedCheckServiceNodes{ - Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc2", "cluster-01", "10.40.1.1", false)}, + Nodes: structs.CheckServiceNodes{structs.TestCheckNodeServiceWithNameInPeer(t, "db", "dc2", "cluster-01", "10.40.1.1", false, uid.EnterpriseMeta)}, }, }) case "failover-through-double-remote-gateway-triggered": @@ -256,6 +261,7 @@ func setupTestVariationConfigEntriesAndSnapshot( func setupTestVariationDiscoveryChain( t testing.T, variation string, + enterprise bool, entMeta acl.EnterpriseMeta, additionalEntries ...structs.ConfigEntry, ) *structs.CompiledDiscoveryChain { @@ -343,6 +349,14 @@ func setupTestVariationDiscoveryChain( }, ) case "failover-to-cluster-peer": + target := structs.ServiceResolverFailoverTarget{ + Peer: "cluster-01", + } + + if enterprise { + target.Namespace = "ns9" + } + entries = append(entries, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, @@ -352,14 +366,19 @@ func setupTestVariationDiscoveryChain( RequestTimeout: 33 * time.Second, Failover: map[string]structs.ServiceResolverFailover{ "*": { - Targets: []structs.ServiceResolverFailoverTarget{ - {Peer: "cluster-01"}, - }, + Targets: []structs.ServiceResolverFailoverTarget{target}, }, }, }, ) case "redirect-to-cluster-peer": + redirect := &structs.ServiceResolverRedirect{ + Peer: "cluster-01", + } + if enterprise { + redirect.Namespace = "ns9" + } + entries = append(entries, &structs.ServiceResolverConfigEntry{ Kind: structs.ServiceResolver, @@ -367,9 +386,7 @@ func setupTestVariationDiscoveryChain( EnterpriseMeta: entMeta, ConnectTimeout: 33 * time.Second, RequestTimeout: 33 * time.Second, - Redirect: &structs.ServiceResolverRedirect{ - Peer: "cluster-01", - }, + Redirect: redirect, }, ) case "failover-through-double-remote-gateway-triggered": diff --git a/agent/proxycfg/upstreams.go b/agent/proxycfg/upstreams.go index 23fab4d2c0f5..b42b5e286e21 100644 --- a/agent/proxycfg/upstreams.go +++ b/agent/proxycfg/upstreams.go @@ -314,7 +314,7 @@ func (s *handlerUpstreams) resetWatchesFromChain( } opts := targetWatchOpts{upstreamID: uid} - opts.fromChainTarget(chain, target) + opts.fromChainTarget(target) err := s.watchUpstreamTarget(ctx, snap, opts) if err != nil { @@ -432,30 +432,13 @@ type targetWatchOpts struct { entMeta *acl.EnterpriseMeta } -func (o *targetWatchOpts) fromChainTarget(c *structs.CompiledDiscoveryChain, t *structs.DiscoveryTarget) { +func (o *targetWatchOpts) fromChainTarget(t *structs.DiscoveryTarget) { o.chainID = t.ID o.service = t.Service o.filter = t.Subset.Filter o.datacenter = t.Datacenter o.peer = t.Peer o.entMeta = t.GetEnterpriseMetadata() - - // The peer-targets in a discovery chain intentionally clear out - // the partition field, since we don't know the remote service's partition. - // Therefore, we must query with the chain's local partition / DC, or else - // the services will not be found. - // - // Note that the namespace is not swapped out, because it should - // always match the value in the remote datacenter (and shouldn't - // have been changed anywhere). - if o.peer != "" { - o.datacenter = "" - // Clone the enterprise meta so it's not modified when we swap the partition. - var em acl.EnterpriseMeta - em.Merge(o.entMeta) - em.OverridePartition(c.Partition) - o.entMeta = &em - } } func (s *handlerUpstreams) watchUpstreamTarget(ctx context.Context, snap *ConfigSnapshotUpstreams, opts targetWatchOpts) error { @@ -470,9 +453,6 @@ func (s *handlerUpstreams) watchUpstreamTarget(ctx context.Context, snap *Config if opts.peer != "" { uid = NewUpstreamIDFromTargetID(opts.chainID) - // chainID has the partition stripped. However, when a target is in a cluster peer, the partition should be set - // to the local partition (i.e chain.Partition), since the peered target is imported into the local partition. - uid.OverridePartition(opts.entMeta.PartitionOrDefault()) correlationID = upstreamPeerWatchIDPrefix + uid.String() } diff --git a/agent/structs/discovery_chain.go b/agent/structs/discovery_chain.go index cb2c1b6f1f98..75ae6c25dbba 100644 --- a/agent/structs/discovery_chain.go +++ b/agent/structs/discovery_chain.go @@ -315,7 +315,7 @@ func (t *DiscoveryTarget) ToDiscoveryTargetOpts() DiscoveryTargetOpts { func ChainID(opts DiscoveryTargetOpts) string { // NOTE: this format is similar to the SNI syntax for simplicity if opts.Peer != "" { - return fmt.Sprintf("%s.%s.default.external.%s", opts.Service, opts.Namespace, opts.Peer) + return fmt.Sprintf("%s.%s.%s.external.%s", opts.Service, opts.Namespace, opts.Partition, opts.Peer) } if opts.ServiceSubset == "" { return fmt.Sprintf("%s.%s.%s.%s", opts.Service, opts.Namespace, opts.Partition, opts.Datacenter) diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index f7f2a7e710e8..ff36545b66a1 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -1,6 +1,9 @@ package structs import ( + "fmt" + + "github.com/hashicorp/consul/acl" "github.com/mitchellh/go-testing-interface" ) @@ -55,7 +58,14 @@ func TestNodeServiceWithName(t testing.T, name string) *NodeService { const peerTrustDomain = "1c053652-8512-4373-90cf-5a7f6263a994.consul" -func TestCheckNodeServiceWithNameInPeer(t testing.T, name, dc, peer, ip string, useHostname bool) CheckServiceNode { +func TestCheckNodeServiceWithNameInPeer(t testing.T, name, dc, peer, ip string, useHostname bool, remoteEntMeta acl.EnterpriseMeta) CheckServiceNode { + + // Non-default partitions have a different spiffe format. + spiffe := fmt.Sprintf("spiffe://%s/ns/default/dc/%s/svc/%s", peerTrustDomain, dc, name) + if !remoteEntMeta.InDefaultPartition() { + spiffe = fmt.Sprintf("spiffe://%s/ap/%s/ns/%s/dc/%s/svc/%s", + peerTrustDomain, remoteEntMeta.PartitionOrDefault(), remoteEntMeta.NamespaceOrDefault(), dc, name) + } service := &NodeService{ Kind: ServiceKindTypical, Service: name, @@ -66,11 +76,10 @@ func TestCheckNodeServiceWithNameInPeer(t testing.T, name, dc, peer, ip string, Connect: ServiceConnect{ PeerMeta: &PeeringServiceMeta{ SNI: []string{ - name + ".default.default." + peer + ".external." + peerTrustDomain, - }, - SpiffeID: []string{ - "spiffe://" + peerTrustDomain + "/ns/default/dc/" + peer + "-dc/svc/" + name, + fmt.Sprintf("%s.%s.%s.%s.external.%s", + name, remoteEntMeta.NamespaceOrDefault(), remoteEntMeta.PartitionOrDefault(), peer, peerTrustDomain), }, + SpiffeID: []string{spiffe}, Protocol: "tcp", }, }, diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 80526c65e51f..2779fd2da781 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -1308,10 +1308,6 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( targetUID := proxycfg.NewUpstreamIDFromTargetID(targetData.targetID) if targetUID.Peer != "" { - // targetID already has a stripped partition, so targetUID will not have a partition either. However, - // when a failover target is in a cluster peer, the partition should be set to the local partition (i.e - // chain.Partition), since that's where the data is imported to. - targetUID.OverridePartition(chain.Partition) peerMeta, found := upstreamsSnapshot.UpstreamPeerMeta(targetUID) if !found { s.Logger.Warn("failed to fetch upstream peering metadata for cluster", "target", targetUID) diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden index 78d23803d12a..aed47665204b 100644 --- a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-failover-to-cluster-peer.latest.golden @@ -1,209 +1,183 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "clusterType": { - "name": "envoy.clusters.aggregate", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig", - "clusters": [ + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "clusterType": { + "name": "envoy.clusters.aggregate", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig", + "clusters": [ "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "failover-target~db.default.cluster-01.external.peer1.domain" ] } }, - "connectTimeout": "33s", - "lbPolicy": "CLUSTER_PROVIDED" + "connectTimeout": "33s", + "lbPolicy": "CLUSTER_PROVIDED" }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "failover-target~db.default.cluster-01.external.peer1.domain", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~db.default.cluster-01.external.peer1.domain", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "1s", - "circuitBreakers": { - - }, - "outlierDetection": { - "maxEjectionPercent": 100 + "connectTimeout": "1s", + "circuitBreakers": {}, + "outlierDetection": { + "maxEjectionPercent": 100 }, - "commonLbConfig": { - "healthyPanicThreshold": { - - } + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "peer1-root-1\n" + "validationContext": { + "trustedCa": { + "inlineString": "peer1-root-1\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/dc2/svc/db" } ] } }, - "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "altStatName": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "33s", - "circuitBreakers": { - - }, - "outlierDetection": { - + "connectTimeout": "33s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "commonLbConfig": { - "healthyPanicThreshold": { - - } - }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" } ] } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "5s", - "circuitBreakers": { - - }, - "outlierDetection": { - - }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" }, { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" } ] } }, - "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "local_app", - "type": "STATIC", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "local_app", - "endpoints": [ + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ { - "lbEndpoints": [ + "lbEndpoints": [ { - "endpoint": { - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 8080 + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 } } } @@ -214,6 +188,6 @@ } } ], - "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden index a797d7e89142..70b94debf208 100644 --- a/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/connect-proxy-with-chain-and-redirect-to-cluster-peer.latest.golden @@ -1,134 +1,118 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "db.default.cluster-01.external.peer1.domain", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.cluster-01.external.peer1.domain", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "1s", - "circuitBreakers": { - - }, - "outlierDetection": { - "maxEjectionPercent": 100 + "connectTimeout": "1s", + "circuitBreakers": {}, + "outlierDetection": { + "maxEjectionPercent": 100 }, - "commonLbConfig": { - "healthyPanicThreshold": { - - } + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "peer1-root-1\n" + "validationContext": { + "trustedCa": { + "inlineString": "peer1-root-1\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/dc2/svc/db" } ] } }, - "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "5s", - "circuitBreakers": { - - }, - "outlierDetection": { - - }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" }, { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" } ] } }, - "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "local_app", - "type": "STATIC", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "local_app", - "endpoints": [ + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ { - "lbEndpoints": [ + "lbEndpoints": [ { - "endpoint": { - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 8080 + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 } } } @@ -139,6 +123,6 @@ } } ], - "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden b/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden index f0f79211fa88..fbdfa6eede93 100644 --- a/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden +++ b/agent/xds/testdata/clusters/ingress-with-chain-and-failover-to-cluster-peer.latest.golden @@ -1,142 +1,122 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "clusterType": { - "name": "envoy.clusters.aggregate", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig", - "clusters": [ + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "clusterType": { + "name": "envoy.clusters.aggregate", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig", + "clusters": [ "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "failover-target~db.default.cluster-01.external.peer1.domain" ] } }, - "connectTimeout": "33s", - "lbPolicy": "CLUSTER_PROVIDED", - "outlierDetection": { - - } + "connectTimeout": "33s", + "lbPolicy": "CLUSTER_PROVIDED", + "outlierDetection": {} }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "failover-target~db.default.cluster-01.external.peer1.domain", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~db.default.cluster-01.external.peer1.domain", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "33s", - "circuitBreakers": { - + "connectTimeout": "33s", + "circuitBreakers": {}, + "outlierDetection": { + "maxEjectionPercent": 100 }, - "outlierDetection": { - "maxEjectionPercent": 100 - }, - "commonLbConfig": { - "healthyPanicThreshold": { - - } + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "peer1-root-1\n" + "validationContext": { + "trustedCa": { + "inlineString": "peer1-root-1\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cluster-01-dc/svc/db" + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/dc2/svc/db" } ] } }, - "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" + "sni": "db.default.default.cluster-01.external.1c053652-8512-4373-90cf-5a7f6263a994.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "altStatName": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": { - - }, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "failover-target~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "33s", - "circuitBreakers": { - - }, - "outlierDetection": { - - }, - "commonLbConfig": { - "healthyPanicThreshold": { - - } + "connectTimeout": "33s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": { - - }, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" } ] } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } } ], - "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" } \ No newline at end of file From a3d05b61a0cf57ad5550c8cdaef3b31ec66b587e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 24 Mar 2023 16:15:54 -0700 Subject: [PATCH 129/421] Backport of Docs/intentions refactor docs day 2022 into release/1.15.x (#16775) * backport of commit 945c13236db5b8d746116ace4468bc66e7a04af2 * backport of commit 4034c6f753a5023fdaacd28b67f86ffb52dc1206 * backport of commit 8c06a1883e2023a3e68ec10e2edd83b684acc9c0 * backport of commit 35757aa1f602018379dbc5fbf1e4c5ccfbce0624 * backport of commit 1204b419ac7d4a46d6ac4cad976d6992c46d3121 * Docs/intentions refactor docs day 2022 (#16758) * converted intentions conf entry to ref CT format * set up intentions nav * add page for intentions usage * final intentions usage page * final intentions overview page * fixed old relative links * updated diagram for overview * updated links to intentions content * fixed typo in updated links * rename intentions overview page file to index * rollback link updates to intentions overview * fixed nav * Updated custom HTML in API and CLI pages to MD * applied suggestions from review to index page * moved conf examples from usage to conf ref * missed custom HTML section * applied additional feedback * Apply suggestions from code review Co-authored-by: Tu Nguyen * updated headings in usage page * renamed files and udpated nav * updated links to new file names * added redirects and final tweaks * typo --------- Co-authored-by: Tu Nguyen * remove old files --------- Co-authored-by: trujillo-adam Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen --- .../content/api-docs/connect/intentions.mdx | 110 +- website/content/commands/connect/envoy.mdx | 2 +- website/content/commands/connect/index.mdx | 2 +- website/content/commands/intention/check.mdx | 11 +- website/content/commands/intention/create.mdx | 11 +- website/content/commands/intention/delete.mdx | 11 +- website/content/commands/intention/get.mdx | 11 +- website/content/commands/intention/index.mdx | 7 +- website/content/commands/intention/list.mdx | 11 +- website/content/commands/intention/match.mdx | 11 +- .../config-entries/service-defaults.mdx | 4 +- .../config-entries/service-intentions.mdx | 1363 +++++++++++------ website/content/docs/connect/index.mdx | 26 +- website/content/docs/connect/intentions.mdx | 341 ----- .../intentions/create-manage-intentions.mdx | 178 +++ .../content/docs/connect/intentions/index.mdx | 91 ++ .../legacy.mdx} | 3 +- website/content/docs/k8s/connect/index.mdx | 8 +- .../docs/lambda/invoke-from-lambda.mdx | 2 +- .../content/docs/security/acl/acl-rules.mdx | 2 +- website/data/docs-nav-data.json | 25 +- ...onsul-service-mesh-intentions-overview.svg | 1 + website/redirects.js | 10 + 23 files changed, 1269 insertions(+), 972 deletions(-) delete mode 100644 website/content/docs/connect/intentions.mdx create mode 100644 website/content/docs/connect/intentions/create-manage-intentions.mdx create mode 100644 website/content/docs/connect/intentions/index.mdx rename website/content/docs/connect/{intentions-legacy.mdx => intentions/legacy.mdx} (98%) create mode 100644 website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg diff --git a/website/content/api-docs/connect/intentions.mdx b/website/content/api-docs/connect/intentions.mdx index f5b38e082cf4..1658b16992fb 100644 --- a/website/content/api-docs/connect/intentions.mdx +++ b/website/content/api-docs/connect/intentions.mdx @@ -43,16 +43,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------ | -| `NO` | `none` | `none` | `intentions:write`1 | - -

     +| `NO` | `none` | `none` | `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention create -replace`](/consul/commands/intention/create#replace). @@ -149,16 +140,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------ | -| `NO` | `none` | `none` | `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `NO` | `none` | `none` | `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention create`](/consul/commands/intention/create). @@ -246,16 +228,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------ | -| `NO` | `none` | `none` | `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `NO` | `none` | `none` | `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | This endpoint supports the same parameters as the [create an intention](#create-intention-with-id) endpoint. Additional parameters unique to this endpoint include: @@ -300,16 +273,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ----------------------------- | -| `YES` | `all` | `none` | `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `YES` | `all` | `none` | `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention get`](/consul/commands/intention/get). @@ -372,16 +336,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ----------------------------- | -| `YES` | `all` | `none` | `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `YES` | `all` | `none` | `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention get`](/consul/commands/intention/get). @@ -435,16 +390,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ----------------------------- | -| `YES` | `all` | `none` | `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `YES` | `all` | `none` | `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention list`](/consul/commands/intention/list). @@ -522,16 +468,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------ | -| `NO` | `none` | `none` | `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `NO` | `none` | `none` | `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention delete`](/consul/commands/intention/delete). @@ -577,16 +514,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ------------------------------ | -| `NO` | `none` | `none` | `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `NO` | `none` | `none` | `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention delete`](/consul/commands/intention/delete). @@ -633,16 +561,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ----------------------------- | -| `NO` | `none` | `none` | `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `NO` | `none` | `none` | `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention check`](/consul/commands/intention/check). @@ -693,16 +612,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | -------------------- | ----------------------------- | -| `YES` | `all` | `background refresh` | `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `YES` | `all` | `background refresh` | `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | The corresponding CLI command is [`consul intention match`](/consul/commands/intention/match). diff --git a/website/content/commands/connect/envoy.mdx b/website/content/commands/connect/envoy.mdx index c35a0b4feaa8..c9e0ffb2e932 100644 --- a/website/content/commands/connect/envoy.mdx +++ b/website/content/commands/connect/envoy.mdx @@ -56,7 +56,7 @@ Usage: `consul connect envoy [options] [-- pass-through options]` ACL token from `-token` or the environment and so should be handled as a secret. This token grants the identity of any service it has `service:write` permission for and so can be used to access any upstream service that that service is - allowed to access by [Connect intentions](/consul/docs/connect/intentions). + allowed to access by [service mesh intentions](/consul/docs/connect/intentions). - `-envoy-version` - The version of envoy that is being started. Default is `1.23.1`. This is required so that the correct configuration can be generated. diff --git a/website/content/commands/connect/index.mdx b/website/content/commands/connect/index.mdx index 166b8ff83526..b390f1899354 100644 --- a/website/content/commands/connect/index.mdx +++ b/website/content/commands/connect/index.mdx @@ -10,7 +10,7 @@ description: >- Command: `consul connect` The `connect` command is used to interact with Consul -[Connect](/consul/docs/connect/intentions) subsystems. It exposes commands for +[service mesh](/consul/docs/connect) subsystems. It exposes commands for running the built-in mTLS proxy and viewing/updating the Certificate Authority (CA) configuration. This command is available in Consul 1.2 and later. diff --git a/website/content/commands/intention/check.mdx b/website/content/commands/intention/check.mdx index d9a4d5d328c3..fadc8d224bd9 100644 --- a/website/content/commands/intention/check.mdx +++ b/website/content/commands/intention/check.mdx @@ -31,16 +31,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | ## Usage diff --git a/website/content/commands/intention/create.mdx b/website/content/commands/intention/create.mdx index 31f2ae899a9b..71c491b06ad8 100644 --- a/website/content/commands/intention/create.mdx +++ b/website/content/commands/intention/create.mdx @@ -25,16 +25,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ------------------------------ | -| `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | ## Usage diff --git a/website/content/commands/intention/delete.mdx b/website/content/commands/intention/delete.mdx index 87d9d65e9051..ad2fca1c97e2 100644 --- a/website/content/commands/intention/delete.mdx +++ b/website/content/commands/intention/delete.mdx @@ -19,16 +19,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ------------------------------ | -| `intentions:write`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:write`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | -> **Deprecated** - The one argument form of this command is deprecated in Consul 1.9.0. Intentions no longer need IDs when represented as diff --git a/website/content/commands/intention/get.mdx b/website/content/commands/intention/get.mdx index 68dd744404f6..84b3a86065d0 100644 --- a/website/content/commands/intention/get.mdx +++ b/website/content/commands/intention/get.mdx @@ -24,16 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | ## Usage diff --git a/website/content/commands/intention/index.mdx b/website/content/commands/intention/index.mdx index 9246ef9e0508..d3e4fc56c431 100644 --- a/website/content/commands/intention/index.mdx +++ b/website/content/commands/intention/index.mdx @@ -14,10 +14,9 @@ The `intention` command is used to interact with Connect creating, updating, reading, deleting, checking, and managing intentions. This command is available in Consul 1.2 and later. -Intentions are managed primarily via -[`service-intentions`](/consul/docs/connect/config-entries/service-intentions) config -entries after Consul 1.9. Intentions may also be managed via the [HTTP -API](/consul/api-docs/connect/intentions). +Use the +[`service-intentions`](/consul/docs/connect/config-entries/service-intentions) configuration entry or the [HTTP +API](/consul/api-docs/connect/intentions) to manage intentions. ~> **Deprecated** - This command is deprecated in Consul 1.9.0 in favor of using the [config entry CLI command](/consul/commands/config/write). To create an diff --git a/website/content/commands/intention/list.mdx b/website/content/commands/intention/list.mdx index 928f416095d8..85a7d6b9d6ce 100644 --- a/website/content/commands/intention/list.mdx +++ b/website/content/commands/intention/list.mdx @@ -19,16 +19,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | ## Usage diff --git a/website/content/commands/intention/match.mdx b/website/content/commands/intention/match.mdx index e0bbfc1222e5..4c727b1fc21c 100644 --- a/website/content/commands/intention/match.mdx +++ b/website/content/commands/intention/match.mdx @@ -24,16 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

    +| `intentions:read`

    Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

    | ## Usage diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index e1e15cf7acb2..75b8497c98c2 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -459,7 +459,7 @@ Specifies the default protocol for the service. In service mesh use cases, the ` - [observability](/consul/docs/connect/observability) - [service splitter configuration entry](/consul/docs/connect/config-entries/service-splitter) - [service router configuration entry](/consul/docs/connect/config-entries/service-router) -- [L7 intentions](/consul/docs/connect/intentions) +- [L7 intentions](/consul/docs/connect/intentions/index#l7-traffic-intentions) You can set the global protocol for proxies in the [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#default-protocol) configuration entry, but the protocol specified in the `service-defaults` configuration entry overrides the `proxy-defaults` configuration. @@ -831,7 +831,7 @@ Specifies the default protocol for the service. In service service mesh use case - [observability](/consul/docs/connect/observability) - [`service-splitter` configuration entry](/consul/docs/connect/config-entries/service-splitter) - [`service-router` configuration entry](/consul/docs/connect/config-entries/service-router) -- [L7 intentions](/consul/docs/connect/intentions) +- [L7 intentions](/consul/docs/connect/intentions/index#l7-traffic-intentions) You can set the global protocol for proxies in the [`ProxyDefaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults#default-protocol), but the protocol specified in the `ServiceDefaults` configuration entry overrides the `ProxyDefaults` configuration. diff --git a/website/content/docs/connect/config-entries/service-intentions.mdx b/website/content/docs/connect/config-entries/service-intentions.mdx index 34c948ac1efd..1eb2c42667a0 100644 --- a/website/content/docs/connect/config-entries/service-intentions.mdx +++ b/website/content/docs/connect/config-entries/service-intentions.mdx @@ -1,40 +1,932 @@ --- layout: docs -page_title: Service Intentions - Configuration Entry Reference +page_title: Service intentions configuration entry reference description: >- - The service intentions configuration entry kind defines the communication permissions between service types. Use the reference guide to learn about `""service-intentions""` config entry parameters and how to authorize L4 and L7 communication int he service mesh with intentions. + Use the service intentions configuration entry to allow or deny traffic to services in the mesh from specific sources. Learn how to configure `service-intention` config entries --- -# Service Intentions Configuration Entry ((#service-intentions)) +# Service intentions configuration entry reference --> **1.9.0+:** This config entry is available in Consul versions 1.9.0 and newer. +This topic provides reference information for the service intentions configuration entry. Intentions are configurations for controlling access between services in the service mesh. A single service intentions configuration entry specifies one destination service and one or more L4 traffic sources, L7 traffic sources, or combination of traffic sources. Refer to [Service mesh intentions overview](/consul/docs/connect/intentions) for additional information. -The `service-intentions` config entry kind (`ServiceIntentions` on Kubernetes) controls Connect traffic -authorization for both networking layer 4 (e.g. TCP) and networking layer 7 -(e.g. HTTP). +## Configuration model -Service intentions config entries represent a collection of -[intentions](/consul/docs/connect/intentions) sharing a specific destination. All -intentions governing access to a specific destination are stored in a single -`service-intentions` config entry. + -A single config entry may define a mix of both L4 and L7 style intentions, but -for a specific source L4 and L7 intentions are mutually exclusive. Only one -will apply at a time. Default behavior for L4 is configurable (regardless of -global setting) by defining a low precedence intention for that destination. + -## Interaction with other Config Entries +- [`Kind`](#kind): string | required | must be set to `service-intentions` +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string | `default` | +- [`Partition`](#partition): string | `default` | +- [`Meta`](#meta): map | no default +- [`Sources`](#sources): list | no default + - [`Name`](#sources-name): string | no default + - [`Peer`](#sources-peer): string | no default + - [`Namespace`](#sources-namespace): string | no default | + - [`Partition`](#sources-partition): string | no default | + - [`Action`](#sources-action): string | no default | required for L4 intentions + - [`Permissions`](#sources-permissions): list | no default + - [`Action`](#sources-permissions-action): string | no default | required + - [`HTTP`](#sources-permissions-http): map | required + - [`PathExact`](#sources-permissions-http): string | no default + - [`PathPrefix`](#sources-permissions-http): string | no default + - [`PathRegex`](#sources-permissions-http): string | no default + - [`Methods`](#sources-permissions-http): list | no default + - [`Header`](#sources-permissions-http-header): list of maps |no default + - [`Name`](#sources-permissions-http-header): string | required + - [`Present`](#sources-permissions-http-header): boolean | `false` + - [`Exact`](#sources-permissions-http-header): string | no default + - [`Prefix`](#sources-permissions-http-header): string | no default + - [`Suffix`](#sources-permissions-http-header): string | no default + - [`Regex`](#sources-permissions-http-header): string | no default + - [`Invert`](#sources-permissions-http-header): boolean | `false` + - [`Precedence`](#sources-precedence): number | no default | _read-only_ + - [`Type`](#sources-type): string | `consul` + - [`Description`](#sources-description): string + - [`LegacyID`](#sources-legacyid): string | no default | _read-only_ + - [`LegacyMeta`](#sources-legacymeta): map | no default | _read-only_ + - [`LegacyCreateTime`](#sources-legacycreatetime): string | no default | _read-only_ + - [`LegacyUpdateTime`](#sources-legacyupdatetime): string | no default | _read-only_ -L7 intentions within a config entry are restricted to only destination services -that define their protocol as HTTP-based via a corresponding -[`service-defaults`](/consul/docs/connect/config-entries/service-defaults) config entry -or globally via [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults) . + + -## Sample Config Entries +- [`apiVersion`](#apiversion): string | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | must be set to `ServiceIntentions` +- [`metadata`](#metadata): map | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | `default` | +- [`spec`](#spec): map | no default + - [`destination`](#spec-destination): map | no default + - [`name`](#spec-destination-name): string | required + - [`namespace`](#metadata-namespace): string | `default` | + - [`sources`](#spec-sources): list | no default + - [`name`](#spec-sources-name): string | no default + - [`peer`](#spec-sources-peer): string | no default + - [`namespace`](#spec-sources-namespace): string | no default | + - [`partition`](#spec-sources-partition): string | no default | + - [`action`](#spec-sources-action): string | no default | required for L4 intentions + - [`permissions`](#spec-sources-permissions): list | no default + - [`action`](#spec-sources-permissions-action): string | no default | required + - [`http`](#spec-sources-permissions-http): map | required + - [`pathExact`](#spec-sources-permissions-http): string | no default + - [`pathPrefix`](#spec-sources-permissions-http): string | no default + - [`pathRegex`](#spec-sources-permissions-http): string | no default + - [`methods`](#spec-sources-permissions-http): list | no default + - [`header`](#spec-sources-permissions-http-header): list of maps |no default + - [`name`](#spec-sources-permissions-http-header): string | required + - [`present`](#spec-sources-permissions-http-header): boolean | `false` + - [`exact`](#spec-sources-permissions-http-header): string | no default + - [`prefix`](#spec-sources-permissions-http-header): string | no default + - [`suffix`](#spec-sources-permissions-http-header): string | no default + - [`regex`](#spec-sources-permissions-http-header): string | no default + - [`invert`](#spec-sources-permissions-http-header): boolean | `false` + - [`type`](#spec-sources-type): string | `consul` + - [`description`](#spec-sources-description): string -The following examples demonstrate potential use-cases for the `service-intentions` configuration entry. + + -### REST Access +## Complete configuration + + + + + +```hcl +Kind = "service-intentions" +Name = "" +Namespace = "" # string +Partition = "" # string +Meta = { + "" = "" + "" = "" + } +Sources = [ + { + Name = "" # string + Peer = "" # string + Namespace = "" # string + Partition = "" # string + Action = "allow" or "deny" # string for L4 intentions + Permissions = [ + { + Action = "allow" or "deny" # string for L7 intenions + HTTP = { + PathExact = "" # string + PathPrefix = "" # string + PathRegex = "" # string + Methods = [ + "", # string + "" + ] + Header = [ + { + Name = "" # string + Present = # boolean + }, + { + Name = "" # string + Exact = "" # boolean + }, + { + Name = "" # string + Prefix = "" # string + }, + { + Name = "" # string + Suffix = "" # string + }, + { + Name = "" # string + Regex = "" # string + Invert = # boolean + } + ] + } + } + ] + Type = "consul" # string + Description = "" # string + Precedence = # number + LegacyID = # string + LegacyMeta = # string + LegacyCreateTime = # string + LegacyUpdateTime = # string + } +] +``` + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: service-intentions +metadata: + name: + namespace: +spec: + destination: + destination: + name: + namespace: + sources: + name: + peer: + namespace: + partition: + action: allow or deny + permissions: + - action: allow or deny + http: + pathExact: + pathPrefix: + pathRegex: + methods: + - + + header: + - name: + present: true + - name: + exact: false + - name: + prefix: + - name: + suffix: + - name: + regex: + invert: false + type: consul + description: +``` + + + + +```json +{ + "Kind":"service-intentions", + "Name":"", + "Namespace":"", + "Partition":"", + "Meta":{ + "key-1":"", + "key-2":"" + }, + "Sources":[ + { + "Name":"", + "Peer":"", + "Namespace":"", + "Partition":"", + "Action":"allow or deny", + "Permissions":[ + { + "Action":"allow or deny", + "HTTP":{ + "PathExact":"", + "PathPrefix":"", + "PathRegex":"", + "Methods":[ + "", + "" + ], + "Header":[ + { + "Name":"", + "Present":true + }, + { + "Name":"", + "Exact":false + }, + { + "Name":"", + "Prefix":"" + }, + { + "Name":"", + "Suffix":"" + }, + { + "Name":"", + "Regex":"", + "Invert":false + } + ] + } + } + ], + "Type":"consul", + "Description":"", + "Precedence":"", + "LegacyID":"", + "LegacyMeta":"", + "LegacyCreateTime":"", + "LegacyUpdateTime":"" + } + ] +} +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service defaults configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. Must be set to `service-intentions`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `service-intentions`. + +### `Name` + +Specifies a name of the destination service for all intentions defined in the configuration entry. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: String + +You can also specify a wildcard character (`*`) to match all services without intentions. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`Permissions`](#sources-permissions). + +### `Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. Services in the namespace are the traffic destinations that the intentions allow or deny traffic to. + +#### Values + +- Default: `default` +- Data type: String + +You can also specify a wildcard character (`*`) to match all namespaces. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`Permissions`](#sources-permissions). + +### `Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to apply the configuration entry. Services in the specified partition are the traffic destinations that the intentions allow or deny traffic to. + +#### Values + +- Default: `default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store when the configuration entry is evaluated. + +#### Values + +- Default: None +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `Sources[]` + +List of configurations that define intention sources and the authorization granted to the sources. You can specify source configurations in any order, but Consul stores and evaluates them in order of reverse precedence at runtime. Refer to [`Precedence`](#sources-precedence) for additional information. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `Name` + - `Peer` + - `Namespace` + - `Partition` + - `Action` + - `Permissions` + - `Precedence` + - `Type` + - `Description` + - `LegacyID` + - `LegacyMeta` + - `LegacyCreateTime` + - `LegacyUpdateTime` + +### `Sources[].Name` + +Specifies the name of the source that the intention allows or denies traffic from. If [`Type`](#sources-type) is set to `consul`, then the value refers to the name of a Consul service. The source is not required to be registered into the Consul catalog. + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `Sources[].Peer` + +Specifies the name of a peered Consul cluster that the intention allows or denies traffic from. Refer to [Cluster peering overview](/consul/docs/connect/cluster-peering) for additional information about peers. + +The `Peer` and `Partition` fields are mutually exclusive. + +#### Values + +- Default: None +- Data type: String + +### `Sources[].Namespace` + +Specifies the traffic source namespace that the intention allows or denies traffic from. + +#### Values + +- Default: If [`Peer`](#sources-peer) is unspecified, defaults to the destination [`Namespace`](#namespace). +- Data type: String + +### `Sources[].Partition` + +Specifies the name of an admin partition that the intention allows or denies traffic from. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information about partitions. + +The `Peer` and `Partition` fields are mutually exclusive. + +#### Values + +- Default: If [`Peer`](#sources-peer) is unspecified, defaults to the destination [`Partition`](#partition). +- Data type: string + +### `Sources[].Action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. Do not configure this field to apply L7 intentions to the same source. Configure the [`Permissions`](#sources-permissions) field instead. + +#### Values + +- Default: None +- This field is required for L4 intentions. +- Data type: String value set to either `allow` or `deny` + +Refer to the following examples for additional guidance: + +- [L4 Intentions for specific sources and destinations](#l4-intentions-for-specific-sources-and-destinations) +- [L4 intentions for all destinations](#l4-intentions-for-all-destinations) +- [L4 intentions for all sources](#l4-intentions-for-all-sources) +- [L4 and L7](#l4-and-l7) + +### `Sources[].Permissions[]` + +Specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. + +Consul applies permissions in the order specified in the configuration. Beginning at the top of the list, Consul applies the first matching request and stops evaluating against the remaining configurations. + +For requests that do not match any of the defined permissions, Consul applies the intention behavior defined in the [`acl_default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. + +Do not configure this field for L4 intentions. Use the [`Sources.Action`](#sources-action) parameter instead. + +The `Permissions` only applies to services with a compatible protocol. `Permissions` are not supported when the [`Name`](#name) or [`Namespace`](#namespace) field is configured with a wildcard because service instances or services in a namespace may use different protocols. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `Action` + - `HTTP` + +Refer to the following examples for additional guidance: + +- [Rest access](#rest-access) +- [gRPC](#grpc) +- [Cluster peering](#cluster-peering) +- [L4 and L7](#l4-and-l7) + +### `Sources[].Permissions[].Action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value set to either `allow` or `deny`. + +### `Sources[].Permissions[].HTTP` + +Specifies a set of HTTP-specific match criteria. Consul applies the action defined in the [`Action`](#sources-permissions-action) field to source traffic that matches the criteria. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +The following table describes the parameters that the HTTP map may contain: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `PathExact` | Specifies an exact path to match on the HTTP request path. Do not specify `PathExact` if `PathPrefix` or `PathRegex` are configured in the same `HTTP` configuration. | string | none | +| `PathPrefix` | Specifies a path prefix to match on the HTTP request path. Do not specify `PathPrefix` if `PathExact` or `PathRegex` are configured in the same `HTTP` configuration. | string | none | +| `PathRegex` | Defines a regular expression to match on the HTTP request path. Do not specify `PathRegex` if `PathExact` or `PathPrefix` are configured in the same `HTTP` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | none | +| `Methods` | Specifies a list of HTTP methods. Consul applies the permission if a request matches the `PathExact`, `PathPrefix`, `PathRegex`, or `Header`, and the source sent the request using one of the specified methods. Refer to the [Mozilla documentation](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) for a list of supported request headers. | list | All request methods | +| `Header` | Specifies a header name and matching criteria for HTTP request headers. Refer to [`Sources[].Permissions[].HTTP[].Header`](#sources-permissions-http-header) for details. | list of maps | none | + +### `Sources[].Permissions[].HTTP[].Header[]` + +Specifies a header name and matching criteria for HTTP request headers. The request header must match all specified criteria for the permission to apply. + +#### Values + +- Default: None +- Data type: list of objects + +Each member of the `Header` list is a map that contains a `Name` field and at least one match criterion. The following table describes the parameters that each member of the `Header` list may contain: + +| Parameter | Description | Data type | Required | +| --- | --- | --- | --- | +| `Name` | Specifies the name of the header to match. | string | required | +| `Present` | Enables a match if the header configured in the `Name` field appears in the request. Consul matches on any value as long as the header key appears in the request. Do not specify `Present` if `Exact`, `Prefix`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | boolean | optional | +| `Exact` | Specifies a value for the header key set in the `Name` field. If the request header value matches the `Exact` value, Consul applies the permission. Do not specify `Exact` if `Present`, `Prefix`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Prefix` | Specifies a prefix value for the header key set in the `Name` field. If the request header value starts with the `Prefix` value, Consul applies the permission. Do not specify `Prefix` if `Present`, `Exact`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Suffix` | Specifies a suffix value for the header key set in the `Name` field. If the request header value ends with the `Suffix` value, Consul applies the permission. Do not specify `Suffix` if `Present`, `Exact`, `Prefix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Regex` | Specifies a regular expression pattern as the value for the header key set in the `Name` field. If the request header value matches the regex, Consul applies the permission. Do not specify `Regex` if `Present`, `Exact`, `Prefix`, or `Suffix` are configured in the same `Header` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | optional | +| `Invert` | Inverts the matching logic configured in the `Header`. Default is `false`. | boolean | optional | + + +### `Sources[].Precedence` + +The `Precedence` field contains a read-only integer. Consul generates the value based on name configurations for the source and destination services. Refer to [Precedence and matching order](/consul/docs/connect/intentions/create-manage-intentions#precedence-and-matching-order) for additional information. + +### `Sources[].Type` + +Specifies the type of destination service that the configuration entry applies to. The only value supported is `consul`. + +#### Values + +- Default: `consul` +- Data type: String + +### `Sources[].Description` + +Specifies a description of the intention. Consul presents the description in API responses to assist other tools integrated into the network. + +#### Values + +- Default: None +- Data type: String + +### `Sources[].LegacyID` + +Read-only unique user ID (UUID) for the intention in the system. Consul generates the value and exposes it in the configuration entry so that legacy API endpoints continue to function. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].LegacyMeta` + +Read-only set of arbitrary key-value pairs to attach to the intention. Consul generates the metadata and exposes it in the configuration entry so that legacy intention API endpoints continue to function. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].CreateTime` + +Read-only timestamp for the intention creation. Consul exposes the timestamp in the configuration entry to allow legacy intention API endpoints to continue functioning. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].LegacyUpdateTime` + +Read-only timestamp marking the most recent intention update. Consul exposes the timestamp in the configuration entry to allow legacy intention API endpoints to continue functioning. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + + + + + +### `apiVersion` + +Specifies the version of the Consul API for integrating with Kubernetes. The value must be `consul.hashicorp.com/v1alpha1`. + +#### Values + +- Default: None +- This field is required. +- String value that must be set to `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. Must be set to `ServiceIntentions`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `ServiceIntentions`. + +### `metadata` + +Map that contains an arbitrary name for the configuration entry and the namespace it applies to. + +#### Values + +- Default: None +- Data type: Map + +### `metadata.name` + +Specifies an arbitrary name for the configuration entry. Note that in other configuration entries, the `metadata.name` field specifies the name of the service that the settings apply to. For service intentions, the service that accepts the configurations is the _destination_ and is specified in the [`spec.destination.name`](#spec-destination-name) field. Refer to the following topics for additional information: + +- [ServiceIntentions Special Case (OSS)](/consul/docs/k8s/crds#serviceintentions-special-case) +- [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) + +#### Values + +- Default: None +- Data type: String + +### `metadata.namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +#### Values + +- Default: `default` +- Data type: String + +### `spec` +Map that contains the details about the `ServiceIntentions` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the spec field. All other configurations are children. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +### `spec.destination` + +Map that identifies the destination name and destination namespace that source services are allowed or denied access to. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +### `spec.destination.name` + +Specifies the name of the destination service in the mesh that the intentions apply to. +You can also specify a wildcard character (`*`) to match all services that are missing intention settings. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`permissions`](#spec-sources-permissions). + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `spec.metadata.namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. You can also specify a wildcard character (`*`) to match all namespaces. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`permissions`](#spec-sources-permissions). + +Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +#### Values + +- Default: If not set, destination service namespace is inherited from the `connectInject.consulNamespaces` configuration. Refer to [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for details. +- Data type: String + +### `spec.sources[]` + +List of configurations that define intention sources and the authorization granted to the sources. You can specify source configurations in any order, but Consul stores and evaluates them in order of reverse precedence at runtime. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `name` + - `peer` + - `namespace` + - `partition` + - `Action` + - `permissions` + - `type` + - `description` + +### `spec.sources[].name` + +Specifies the name of the source that the intention allows or denies traffic from. If [`type`](#sources-type) is set to `consul`, then the value refers to the name of a Consul service. The source is not required to be registered into the Consul catalog. + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `spec.sources[].peer` + +Specifies the name of a peered Consul cluster that the intention allows or denies traffic from. Refer to [Cluster peering overview](/consul/docs/connect/cluster-peering) for additional information about peers. The `peer` and `partition` fields are mutually exclusive. +#### Values + +- Default: None +- Data type: String + +### `spec.sources[].namespace` + +Specifies the traffic source namespace that the intention allows or denies traffic from. + +#### Values + +- Default: If [`peer`](#spec-sources-peer) is unspecified, defaults to the namespace specified in the [`spec.destination.namespace`](#spec-destination-namespace) field. +- Data type: String + +### `spec.sources[].partition` + +Specifies the name of an admin partition that the intention allows or denies traffic from. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information about partitions. The `peer` and `partition` fields are mutually exclusive. + +#### Values + +- Default: If [`peer`](#sources-peer) is unspecified, defaults to the partition specified in [`spec.destination.partition`](#spec-destination-partition). +- Data type: String + +### `spec.sources[].action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. Do not configure this field for L7 intentions. Configure the [`spec.sources.permissions`](#spec-sources-permissions) field instead. + +#### Values + +- Default: None +- This field is required for L4 intentions. +- Data type: String value set to either `allow` or `deny` + +### `spec.sources[].permissions[]` + +Specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. + +Consul applies permissions in the order specified in the configuration. Starting at the beginning of the list, Consul applies the first matching request and stops evaluating against the remaining configurations. + +For requests that do not match any of the defined permissions, Consul applies the intention behavior defined in the [`acl_default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. + +Do not configure this field for L4 intentions. Use the [`spec.sources.action`](#sources-action) parameter instead. + +`permissions` configurations only apply to services with a compatible protocol. As a result, they are not supported when the [`spec.destination.name`](#spec-destination-name) or [`spec.destination.namespace`](#spec-destination-namespace) field is configured with a wildcard because service instances or services in a namespace may use different protocols. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `action` + - `http` + +### `spec.sources[].permissions[].action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value set to either `allow` or `deny` + +### `spec.sources[].permissions[].http` + +Specifies a set of HTTP-specific match criteria. Consul applies the action defined in the [`spec.sources.permissions.action`](#spec-sources-permissions-action) field to source traffic that matches the criteria. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +The following table describes the parameters that the HTTP map may contain: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `pathExact` | Specifies an exact path to match on the HTTP request path. Do not specify `pathExact` if `pathPrefix` or `pathRegex` are configured in the same `http` configuration. | string | none | +| `pathPrefix` | Specifies a path prefix to match on the HTTP request path. Do not specify `pathPrefix` if `pathExact` or `pathRegex` are configured in the same `http` configuration. | string | none | +| `pathRegex` | Defines a regular expression to match on the HTTP request path. Do not specify `pathRegex` if `pathExact` or `pathPrefix` are configured in the same `http` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | none | +| `methods` | Specifies a list of HTTP methods. Consul applies the permission if a request matches the `pathExact`, `pathPrefix`, `pathRegex`, or `header`, and the source sent the request using one of the specified methods. Refer to the [Mozilla documentation](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) for a list of supported request headers. | list | All request methods | +| `header` | Specifies a header name and matching criteria for HTTP request headers. Refer to [`spec.sources[].permissions[].http[].header`](#spec-sources-permissions-http-header) for details. | list of maps | none | + +### `spec.sources[].permissions[].http[].header` + +Specifies a set of criteria for matching HTTP request headers. The request header must match all specified criteria for the permission to apply. + +#### Values + +- Default: None +- Data type: List of maps + +Each member of the `header` list is a map that contains a `name` field and at least one match criterion. The following table describes the parameters that each member of the `header` list may contain: + +| Parameter | Description | Data type | Required | +| --- | --- | --- | --- | +| `name` | Specifies the name of the header to match. | string | required | +| `present` | Enables a match if the header configured in the `name` field appears in the request. Consul matches on any value as long as the header key appears in the request. Do not specify `present` if `exact`, `prefix`, `suffix`, or `regex` are configured in the same `header` configuration. | boolean | optional | +| `Exact` | Specifies a value for the header key set in the `Name` field. If the request header value matches the `exact` value, Consul applies the permission. Do not specify `exact` if `present`, `prefix`, `suffix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `prefix` | Specifies a prefix value for the header key set in the `name` field. If the request header value starts with the `prefix` value, Consul applies the permission. Do not specify `prefix` if `present`, `exact`, `suffix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `suffix` | Specifies a suffix value for the header key set in the `name` field. If the request header value ends with the `suffix` value, Consul applies the permission. Do not specify `suffix` if `present`, `exact`, `prefix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `regex` | Specifies a regular expression pattern as the value for the header key set in the `name` field. If the request header value matches the regex, Consul applies the permission. Do not specify `regex` if `present`, `exact`, `prefix`, or `suffix` are configured in the same `header` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | optional | +| `invert` | Inverts the matching logic configured in the `header`. Default is `false`. | boolean | optional | + +### `spec.sources[].type` + +Specifies the type of destination service that the configuration entry applies to. The only value supported is `consul`. + +#### Values + +- Default: `consul` +- Data type: String + +### `spec.sources[].description` + +Specifies a description of the intention. Consul presents the description in API responses to assist other tools integrated into the network. + +#### Values + +- Default: None +- Data type: String + + + + + + +## Examples + +The following examples demonstrate potential use-cases for the service intentions configuration entry. + +### L4 Intentions for specific sources and destinations + +The following example configuration entry specifies an L4 intention that denies traffic from `web` to `db` service instances, but allows traffic from `api` to `db`. + + + +```hcl +Kind = "service-intentions" +Name = "db" +Sources = [ + { + Name = "web" + Action = "deny" + }, + { + Name = "api" + Action = "allow" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +spec: + destination: + name: db + sources: + - name: web + action: deny + - name: api + action: allow +``` + +```json +{ + "Kind": "service-intentions", + "Name": "db", + "Sources": [ + { + "Action": "deny", + "Name": "web" + }, + { + "Action": "allow", + "Name": "api" + } + ] +} +``` + + + +### L4 intentions for all destinations + +In the following L4 example, the destination is configured with a `*` wildcard. As a result, traffic from `web` service instances is denied for any service in the datacenter. + + + +```hcl +Kind = "service-intentions" +Name = "*" +Sources = [ + { + Name = "web" + Action = "deny" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +spec: + destination: + name: * + sources: + - name: web + action: deny +``` + +```json +{ + "Kind": "service-intentions", + "Name": "*", + "Sources": [ + { + "Action": "deny", + "Name": "web" + } + ] +} +``` + + + +### L4 intentions for all sources + +In the following L4 example, the source is configured with a `*` wildcard. As a result, traffic from any service is denied to `db` service instances. + + + +```hcl +Kind = "service-intentions" +Name = "db" +Sources = [ + { + Name = "*" + Action = "deny" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +spec: + destination: + name: db + sources: + - name: * + action: deny +``` + +```json +{ + "Kind": "service-intentions", + "Name": "db", + "Sources": [ + { + "Action": "deny", + "Name": "*" + } + ] +} +``` + + +### REST access In the following example, the `admin-dashboard` and `report-generator` services have different levels of access when making REST calls: @@ -134,8 +1026,8 @@ spec: ### gRPC -In the following use-case, access to the `IssueRefund` gRPC service method is set to `deny`. Because gRPC method calls [are -HTTP/2](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md), an HTTP path-matching rule can be used to control traffic: +In the following example, Consul denies requests from `frontend-web` to the `IssueRefund` gRPC service. +Because gRPC method calls use the [HTTP/2 protocol](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md), you can apply an HTTP path-matching rule to control traffic: @@ -174,8 +1066,8 @@ Sources = [ } ] } - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ] ``` @@ -204,8 +1096,8 @@ spec: - action: allow http: pathPrefix: '/mycompany.BillingService/' - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ``` ```json @@ -249,7 +1141,7 @@ spec: ### L4 and L7 -You can mix and match L4 and L7 intentions per source: +In the following example, Consul enforces application layer intentions that deny requests to `api` from `hackathon-project` but allow requests from `web`. In the same configuration entry, Consul enforces network layer intentions that allow requests from `nightly-reconciler` that send `POST` requests to the `/v1/reconcile-data` HTTP endpoint: @@ -277,8 +1169,8 @@ Sources = [ } ] }, - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ] ``` @@ -301,8 +1193,8 @@ spec: http: pathExact: /v1/reconcile-data methods: ['POST'] - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ``` ```json @@ -384,406 +1276,5 @@ When using cluster peering connections, intentions secure your deployments with ] } ``` - - -## Available Fields - -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - "Specifies the namespaces the config entry will apply to. This may be set to the wildcard character (`*`) to match all services in all namespaces that don't otherwise have intentions defined. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - "Specifies the admin partition on which the configuration entry will apply. Wildcard characters (`*`) are not supported. Admin partitions must specified explicitly.", - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: 'Specifies arbitrary KV metadata pairs.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: - 'Unlike other config entries, the `metadata.name` field is not used to set the name of the service being configured. Instead, that is set in `spec.destination.name`. Thus this name can be set to anything. See [ServiceIntentions Special Case (OSS)](/consul/docs/k8s/crds#serviceintentions-special-case) or [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for more details.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'destination', - hcl: false, - children: [ - { - name: 'name', - hcl: false, - type: 'string: ', - description: - "The name of the destination service for all intentions defined in this config entry. This may be set to the wildcard character (`*`) to match all services that don't otherwise have intentions defined. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - }, - { - name: 'namespace', - hcl: false, - enterprise: true, - type: 'string: ', - description: - "Specifies the namespaces the config entry will apply to. This may be set to the wildcard character (`*`) to match all services in all namespaces that don't otherwise have intentions defined. If not set, the namespace used will depend on the `connectInject.consulNamespaces` configuration. See [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for more details. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - }, - ], - }, - { - name: 'Sources', - type: 'array', - description: `The list of all [intention sources and the authorization granted to those sources](#sourceintention). The order of - this list does not matter, but out of convenience Consul will always store - this reverse sorted by intention precedence, as that is the order that they - will be evaluated at enforcement time.`, - }, - ]} -/> - -### `SourceIntention` - -', - description: { - hcl: - "The source of the intention. For a `Type` of `consul` this is the name of a Consul service. The service doesn't need to be registered.", - yaml: - "The source of the intention. For a `type` of `consul` this is the name of a Consul service. The service doesn't need to be registered.", - }, - }, - { - name: 'Peer', - type: 'string: ""', - description: { - hcl: - "Specifies the [peer](/consul/docs/connect/cluster-peering/index.mdx) of the source service. `Peer` is mutually exclusive with `Partition`.", - yaml: - "Specifies the [peer](/consul/docs/connect/cluster-peering/index.mdx) of the source service. `peer` is mutually exclusive with `partition`.", - }, - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: { - hcl: - "The namespace of the source service. If `Peer` is empty, `Namespace` defaults to the namespace of the destination service (i.e. the config entry's namespace).", - yaml: - 'The namespace of the source service. If `peer` is empty, `namespace` defaults to the namespace of the destination service (i.e. `spec.destination.namespace`).', - }, - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: { - hcl: - "Specifies the admin partition of the source service. If `Peer` is empty, `Partition` defaults to the destination service's partition (i.e. the configuration entry's partition). `Partition` is mutually exclusive with `Peer`.", - yaml: - "Specifies the admin partition of the source service. If `peer` is empty, `partition` defaults to the destination service's partition (i.e. `spec.destination.partition`). `partition` is mutually exclusive with `peer`.", - }, - }, - { - name: 'Action', - type: 'string: ""', - description: { - hcl: - 'For an L4 intention this is required, and should be set to one of `"allow"` or `"deny"` for the action that should be taken if this intention matches a request.' + - '

    This should be omitted for an L7 intention as it is mutually exclusive with the `Permissions` field.', - yaml: - 'For an L4 intention this is required, and should be set to one of `"allow"` or `"deny"` for the action that should be taken if this intention matches a request.' + - '

    This should be omitted for an L7 intention as it is mutually exclusive with the `permissions` field.', - }, - }, - { - name: 'Permissions', - type: 'array', - description: { - hcl: `The list of all [additional L7 attributes](#intentionpermission) that extend the intention match criteria.

    - Permission precedence is applied top to bottom. For any given request the - first permission to match in the list is terminal and stops further - evaluation. As with L4 intentions, traffic that fails to match any of the - provided permissions in this intention will be subject to the default - intention behavior is defined by the default [ACL policy](/consul/docs/agent/config/config-files#acl_default_policy).

    - This should be omitted for an L4 intention as it is mutually exclusive with - the \`Action\` field.

    - Setting \`Permissions\` is not valid if a wildcard is used for the \`Name\` or \`Namespace\` because they can only be - applied to services with a compatible protocol.`, - yaml: `The list of all [additional L7 attributes](#intentionpermission) that extend the intention match criteria.

    - Permission precedence is applied top to bottom. For any given request the - first permission to match in the list is terminal and stops further - evaluation. As with L4 intentions, traffic that fails to match any of the - provided permissions in this intention will be subject to the default - intention behavior is defined by the default [ACL policy](/consul/docs/agent/config/config-files#acl_default_policy).

    - This should be omitted for an L4 intention as it is mutually exclusive with - the \`action\` field.

    - Setting \`permissions\` is not valid if a wildcard is used for the \`spec.destination.name\` or \`spec.destination.namespace\` - because they can only be applied to services with a compatible protocol.`, - }, - }, - { - name: 'Precedence', - type: 'int: ', - description: - 'An [integer precedence value](/consul/docs/connect/intentions#precedence-and-match-order) computed from the source and destination naming components.', - yaml: false, - }, - { - name: 'Type', - type: 'string: "consul"', - description: { - hcl: - 'The type for the `Name` value. This can be only "consul" today to represent a Consul service. If not provided, this will be defaulted to "consul".', - yaml: - 'The type for the `name` value. This can be only "consul" today to represent a Consul service. If not provided, this will be defaulted to "consul".', - }, - }, - { - name: 'Description', - type: 'string: ""', - description: - 'Description for the intention. This is not used by Consul, but is presented in API responses to assist tooling.', - }, - { - name: 'LegacyID', - type: 'string: ', - description: `This is the UUID to uniquely identify - this intention in the system. Cannot be set directly and is exposed here as - an artifact of the config entry migration and is primarily used to allow - legacy intention [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyMeta', - type: 'map: ', - description: `Specified arbitrary KV - metadata pairs attached to the intention, rather than to the enclosing config - entry. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyCreateTime', - type: 'time: optional', - description: `The timestamp that this intention was - created. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyUpdateTime', - type: 'time: optional', - description: `The timestamp that this intention was - last updated. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - ]} -/> - -### `IntentionPermission` - -', - description: - 'This is one of "allow" or "deny" for the action that should be taken if this permission matches a request.', - }, - { - name: 'HTTP', - type: 'IntentionHTTPPermission: ', - description: - 'A set of [HTTP-specific authorization criteria](#intentionhttppermission)', - }, - ]} -/> - -### `IntentionHTTPPermission` - -
    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Exact path to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathPrefix', - type: 'string: ""', - description: { - hcl: - 'Path prefix to match on the HTTP request path.

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Path prefix to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathRegex', - type: 'string: ""', - description: { - hcl: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'Methods', - type: 'array', - description: - 'A list of HTTP methods for which this match applies. If unspecified all HTTP methods are matched. If provided the names must be a valid [method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods).', - }, - { - name: 'Header', - type: 'array', - description: - 'A set of criteria that can match on HTTP request headers. If more than one is configured all must match for the overall match to apply.', - children: [ - { - name: 'Name', - type: 'string: ', - description: 'Name of the header to match', - }, - { - name: 'Present', - type: 'bool: false', - description: { - hcl: - 'Match if the header with the given name is present with any value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is present with any value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Exact', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name is this value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is this value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Prefix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this prefix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this prefix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Suffix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this suffix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this suffix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Regex', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Invert', - type: 'bool: false', - description: 'Inverts the logic of the match', - }, - ], - }, - ]} -/> - -## ACLs - -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). - -Reading a `service-intentions` config entry requires `intentions:read` on the resource. - -Creating, updating, or deleting a `service-intentions` config entry requires -`intentions:write` on the resource. - -Intention ACL rules are specified as part of a `service` rule. See [Intention -Management Permissions](/consul/docs/connect/intentions#intention-management-permissions) for -more details. - -## Regular Expression Syntax - -The actual syntax of the regular expression fields described here is entirely -proxy-specific. -When using [Envoy](/consul/docs/connect/proxies/envoy) as a proxy, the syntax for -these fields is [RE2](https://github.com/google/re2/wiki/Syntax). +     diff --git a/website/content/docs/connect/index.mdx b/website/content/docs/connect/index.mdx index cbd2903a3318..7c7235569d3e 100644 --- a/website/content/docs/connect/index.mdx +++ b/website/content/docs/connect/index.mdx @@ -5,7 +5,7 @@ description: >- Consul’s service mesh makes application and microservice networking secure and observable with identity-based authentication, mutual TLS (mTLS) encryption, and explicit service-to-service authorization enforced by sidecar proxies. Learn how Consul’s service mesh works and get started on VMs or Kubernetes. --- -# Consul Service Mesh +# Consul service mesh Consul Service Mesh provides service-to-service connection authorization and encryption using mutual Transport Layer Security (TLS). Consul Connect is used interchangeably @@ -25,30 +25,30 @@ Review the video below to learn more about Consul Connect from HashiCorp's co-fo height="315" > -## Application Security +## Application security -Connect enables secure deployment best-practices with automatic +Consul service mesh enables secure deployment best-practices with automatic service-to-service encryption, and identity-based authorization. -Connect uses the registered service identity (rather than IP addresses) to +Consul uses the registered service identity, rather than IP addresses, to enforce access control with [intentions](/consul/docs/connect/intentions). This -makes it easier to reason about access control and enables services to be -rescheduled by orchestrators including Kubernetes and Nomad. Intention -enforcement is network agnostic, so Connect works with physical networks, cloud +makes it easier to control access and enables services to be +rescheduled by orchestrators, including Kubernetes and Nomad. Intention +enforcement is network agnostic, so Consul service mesh works with physical networks, cloud networks, software-defined networks, cross-cloud, and more. ## Observability -One of the key benefits of Consul Connect is the uniform and consistent view it can +One of the key benefits of Consul service mesh is the uniform and consistent view it can provide of all the services on your network, irrespective of their different -programming languages and frameworks. When you configure Consul Connect to use -sidecar proxies, those proxies "see" all service-to-service traffic and can -collect data about it. Consul Connect can configure Envoy proxies to collect +programming languages and frameworks. When you configure Consul service mesh to use +sidecar proxies, those proxies see all service-to-service traffic and can +collect data about it. Consul service mesh can configure Envoy proxies to collect layer 7 metrics and export them to tools like Prometheus. Correctly instrumented applications can also send open tracing data through Envoy. -## Getting Started With Consul Service Mesh +## Getting started with Consul service mesh -There are several ways to try Connect in different environments. +Complete the following tutorials try Consul service mesh in different environments: - The [Getting Started with Consul Service Mesh collection](/consul/tutorials/kubernetes-deploy/service-mesh?utm_source=docs) walks you through installing Consul as service mesh for Kubernetes using the Helm diff --git a/website/content/docs/connect/intentions.mdx b/website/content/docs/connect/intentions.mdx deleted file mode 100644 index b3b0371a93a7..000000000000 --- a/website/content/docs/connect/intentions.mdx +++ /dev/null @@ -1,341 +0,0 @@ ---- -layout: docs -page_title: Service Mesh Intentions -description: >- - Intentions define communication permissions in the service mesh between microservices. Learn about configuration basics, wildcard intentions, precedence and match order, and protecting intention management with ACLs. ---- - -# Service Mesh Intentions - --> **1.9.0 and later:** This guide only applies in Consul versions 1.9.0 and -later. The documentation for the legacy intentions system is -[here](/consul/docs/connect/intentions-legacy). - -Intentions define access control for services via Connect and are used to -control which services may establish connections or make requests. Intentions -can be managed via the API, CLI, or UI. - -Intentions are enforced on inbound connections or requests by the -[proxy](/consul/docs/connect/proxies) or within a [natively integrated -application](/consul/docs/connect/native). - -Depending upon the [protocol] in use by the destination service, you can define -intentions to control Connect traffic authorization either at networking layer -4 (e.g. TCP) and application layer 7 (e.g. HTTP): - -- **Identity-based** - All intentions may enforce access based on identities - encoded within [TLS - certificates](/consul/docs/connect/connect-internals#mutual-transport-layer-security-mtls). - This allows for coarse all-or-nothing access control between pairs of - services. These work with for services with any [protocol] as they only - require awareness of the TLS handshake that wraps the opaque TCP connection. - These can also be thought of as **L4 intentions**. - -- **Application-aware** - Some intentions may additionally enforce access based - on [L7 request - attributes](/consul/docs/connect/config-entries/service-intentions#permissions) in - addition to connection identity. These may only be defined for services with - a [protocol] that is HTTP-based. These can also be thought of as **L7 - intentions**. - -At any given point in time, between any pair of services **only one intention -controls authorization**. This may be either an L4 intention or an L7 -intention, but at any given point in time only one of those applies. - -The [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) -should be periodically called to retrieve all relevant intentions for the -target destination. After verifying the TLS client certificate, the cached -intentions should be consulted for each incoming connection/request to -determine if it should be accepted or rejected. - -The default intention behavior is defined by the [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. -If the configuration is set `allow`, then all service mesh Connect connections will be allowed by default. -If is set to `deny`, then all connections or requests will be denied by default. - -## Intention Basics - -You can define a [`service-intentions`](/consul/docs/connect/config-entries/service-intentions) configuration entry to create and manage intentions, as well as manage intentions through the Consul UI. You can also perform some intention-related tasks using the API and CLI commands. Refer to the [API](/consul/api-docs/connect/intentions) and [CLI](/consul/commands/intention) documentation for details. - -The following example shows a `service-intentions` configuration entry that specifies two intentions. Refer to the [`service-intentions`](/consul/docs/connect/config-entries/service-intentions) documentation for the full data model and additional examples. - - - -```hcl -Kind = "service-intentions" -Name = "db" -Sources = [ - { - Name = "web" - Action = "deny" - }, - { - Name = "api" - Action = "allow" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "db", - "Sources": [ - { - "Action": "deny", - "Name": "web" - }, - { - "Action": "allow", - "Name": "api" - } - ] -} -``` - - - -This configuration entry defines two intentions with a common destination of `db`. The -first intention above is a deny intention with a source of `web`. This says -that connections from web to db are not allowed and the connection will be -rejected. The second intention is an allow intention with a source of `api`. -This says that connections from api to db are allowed and connections will be -accepted. - -### Wildcard Intentions - -You can use the `*` wildcard to match service names when defining an intention source or destination. The wildcard matches _any_ value, which enables you to set a wide initial scope when configuring intentions. - -The wildcard is supported in Consul Enterprise `namespace` fields (see [Namespaces](/consul/docs/enterprise/namespaces) for additional information), but it _is not supported_ in `partition` fields (see [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information). - -In the following example, the `web` service cannot connect to _any_ service: - - - -```hcl -Kind = "service-intentions" -Name = "*" -Sources = [ - { - Name = "web" - Action = "deny" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "*", - "Sources": [ - { - "Action": "deny", - "Name": "web" - } - ] -} -``` - - - -The `db` service is configured to deny all connection in the following example: - - - -```hcl -Kind = "service-intentions" -Name = "db" -Sources = [ - { - Name = "*" - Action = "deny" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "db", - "Sources": [ - { - "Action": "deny", - "Name": "*" - } - ] -} -``` - - - - This example grants Prometheus access to any service in -any namespace. - - - -```hcl -Kind = "service-intentions" -Name = "*" -Namespace = "*" -Sources = [ - { - Name = "prometheus" - Namespace = "monitoring" - Action = "allow" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "*", - "Namespace": "*", - "Sources": [ - { - "Action": "allow", - "Name": "prometheus", - "Namespace": "monitoring" - } - ] -} -``` - - - -### Enforcement - -For services that define their [protocol] as TCP, intentions mediate the -ability to **establish new connections**. When an intention is modified, -existing connections will not be affected. This means that changing a -connection from "allow" to "deny" today _will not_ kill the connection. - -For services that define their protocol as HTTP-based, intentions mediate the -ability to **issue new requests**. - -When an intention is modified, requests received after the modification will -use the latest intention rules to enforce access. This means that though -changing a connection from "allow" to "deny" today will not kill the -connection, it will correctly block new requests from being processed. - -## Precedence and Match Order - -Intentions are matched in an implicit order based on specificity, preferring -deny over allow. Specificity is determined by whether a value is an exact -specified value or is the wildcard value `*`. -The full precedence table is shown below and is evaluated -top to bottom, with larger numbers being evaluated first. - -| Source Namespace | Source Name | Destination Namespace | Destination Name | Precedence | -| ---------------- | ----------- | --------------------- | ---------------- | ---------- | -| Exact | Exact | Exact | Exact | 9 | -| Exact | `*` | Exact | Exact | 8 | -| `*` | `*` | Exact | Exact | 7 | -| Exact | Exact | Exact | `*` | 6 | -| Exact | `*` | Exact | `*` | 5 | -| `*` | `*` | Exact | `*` | 4 | -| Exact | Exact | `*` | `*` | 3 | -| Exact | `*` | `*` | `*` | 2 | -| `*` | `*` | `*` | `*` | 1 | - -The precedence value can be read from a -[field](/consul/docs/connect/config-entries/service-intentions#precedence) on the -`service-intentions` configuration entry after it is modified. Precedence cannot be -manually overridden today. - -The numbers in the table above are not stable. Their ordering will remain -fixed but the actual number values may change in the future. - --> - Namespaces are an Enterprise feature. In Consul -OSS the only allowable value for either namespace field is `"default"`. Other -rows in this table are not applicable. - -## Intention Management Permissions - -Intention management can be protected by [ACLs](/consul/docs/security/acl). -Permissions for intentions are _destination-oriented_, meaning the ACLs -for managing intentions are looked up based on the destination value -of the intention, not the source. - -Intention permissions are by default implicitly granted at `read` level -when granting `service:read` or `service:write`. This is because a -service registered that wants to use Connect needs `intentions:read` -for its own service name in order to know whether or not to authorize -connections. The following ACL policy will implicitly grant `intentions:read` -(note _read_) for service `web`. - - - -```hcl -service "web" { - policy = "write" -} -``` - -```json -{ - "service": [ - { - "web": [ - { - "policy": "write" - } - ] - } - ] -} -``` - - - -It is possible to explicitly specify intention permissions. For example, -the following policy will allow a service to be discovered without granting -access to read intentions for it. - -```hcl -service "web" { - policy = "read" - intentions = "deny" -} -``` - -Note that `intentions:read` is required for a token that a Connect-enabled -service uses to register itself or its proxy. If the token used does not -have `intentions:read` then the agent will be unable to resolve intentions -for the service and so will not be able to authorize any incoming connections. - -~> **Security Note:** Explicitly allowing `intentions:write` on the token you -provide to a service instance at registration time opens up a significant -additional vulnerability. Although you may trust the service _team_ to define -which inbound connections they accept, using a combined token for registration -allows a compromised instance to to redefine the intentions which allows many -additional attack vectors and may be hard to detect. We strongly recommend only -delegating `intentions:write` using tokens that are used by operations teams or -orchestrators rather than spread via application config, or only manage -intentions with management tokens. - -## Performance and Intention Updates - -The intentions for services registered with a Consul agent are cached -locally on that agent. They are then updated via a background blocking query -against the Consul servers. - -Supported [proxies] (such as [Envoy]) also cache this data within their own -configuration so that inbound connections or requests require no Consul agent -involvement during authorization. All actions in the data path of connections -happen within the proxy. - -Updates to intentions are propagated nearly instantly to agents since agents -maintain a continuous blocking query in the background for intention updates -for registered services. Proxies similarly use blocking queries to update -their local configurations quickly. - -Because all the intention data is cached locally, the agents or proxies can -fail static. Even if the agents are severed completely from the Consul servers, -or the proxies are severed completely from their local Consul agent, inbound -connection authorization continues to work indefinitely. Changes to intentions -will not be picked up until the partition heals, but will then automatically -take effect when connectivity is restored. - -[protocol]: /consul/docs/connect/config-entries/service-defaults#protocol -[proxies]: /consul/docs/connect/proxies -[envoy]: /consul/docs/connect/proxies/envoy diff --git a/website/content/docs/connect/intentions/create-manage-intentions.mdx b/website/content/docs/connect/intentions/create-manage-intentions.mdx new file mode 100644 index 000000000000..80e68ce890ad --- /dev/null +++ b/website/content/docs/connect/intentions/create-manage-intentions.mdx @@ -0,0 +1,178 @@ +--- +layout: docs +page_title: Create and manage service intentions +description: >- + Learn how to create and manage Consul service mesh intentions using service-intentions config entries, the `consul intentions` command, and `/connect/intentions` API endpoint. +--- + +# Create and manage intentions + +This topic describes how to create and manage service intentions, which are configurations for controlling access between services in the service mesh. + +## Overview + +You can create single intentions or create them in batches using the Consul API, CLI, or UI. You can also define a service intention configuration entry that sets default intentions for all services in the mesh. Refer to [Service intentions overview](/consul/docs/connnect/intentions/intentions) for additional background information about intentions. + +## Requirements + +- At least two services must be registered in the datacenter. +- TLS must be enabled to enforce L4 intentions. Refer to [Encryption](/consul/docs/security/encryption) for additional information. + +### ACL requirements + +Consul grants permissions for creating and managing intentions based on the destination, not the source. When ACLs are enabled, services and operators must present a token linked to a policy that grants read and write permissions to the destination service. + +Consul implicitly grants `intentions:read` permissions to destination services when they are configured with `service:read` or `service:write` permissions. This is so that the services can allow or deny inbound connections when they attempt to join the service mesh. Refer to [Service rules](/consul/docs/security/acl/acl-rules#service-rules) for additional information about configuring ALCs for intentions. + +The default ACL policy configuration determines the default behavior for intentions. If the policy is set to `deny`, then all connections or requests are denied and you must enable them explicitly. Refer to [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) for details. + +## Create an intention + +You can create and manage intentions one at a time using the Consul API, CLI, or UI You can specify one destination or multiple destinations in a single intention. + +### API + +Send a `PUT` request to the `/connect/intentions/exact` HTTP API endpoint and specify the following query parameters: + +- `source`: Service sending the request +- `destination`: Service responding to the request +- `ns`: Namespace of the destination service + +For L4 intentions, you must also specify the intention action in the request payload. + +The following example creates an intention that allows `web` to send request to `db`: + +```shell-session +$ curl --request PUT \ +--data ' { "Action": "allow" } ' \ +http://localhost:8500/v1/connect/intentions/exact\?source\=web\&destination\=db +``` + +Refer to the `/connect/intentions/exact` [HTTP API endpoint documentation](/consul/api-docs/connect/intentions) for additional information request payload parameters. + +For L7 intentions, specify the `Permissions` in the request payload to configure attributes for dynamically enforcing intentions. In the following example payload, Consul allows HTTP GET requests if the request body is empty: + + + +```json +{ + "Permissions": [ + { + "Action": "allow", + "HTTP": { + "Methods": ["GET"], + "Header": [ + { + "Name": "Content-Length", + "Exact": "0" + } + ] + } + } + ] +} + +``` + + + +The `Permissions` object specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. Refer to the [`Sources[].Permissions[]` parameter](/consul/connect/config-entries/service-intentions#source-permissions) in the service intentions configuration entry reference for configuration details. + +To apply the intention, call the endpoint and pass the configuration file containing the attributes to the endpoint: + +```shell-session +$ curl --request PUT \ +--data @payload.json \ +http://localhost:8500/v1/connect/intentions/exact\?source\=svc1\&destination\=sv2 +``` +### CLI + +Use the `consul intention create` command according to the following syntax to create a new intention: + +```shell-session +$ consul intention create - +``` + +The following example creates an intention that allows `web` service instances to connect to `db` service instances: + +```shell-session +$ consul intention create -allow web db +``` + +You can use the asterisk (`*`) wildcard to specify multiple destination services. Refer to [Precedence and match order](/consul/docs/connect/intentions/create-manage-intentions#precedence-and-match-order) for additional information. + +### Consul UI + +1. Log into the Consul UI and choose **Services** from the sidebar menu. +1. Click on a service and then click the **Intentions* tab. +1. Click **Create** and choose the source service from the drop-down menu. +1. You can add an optional description. +1. Choose one of the following options: + 1. **Allow**: Allows the source service to send requests to the destination. + 1. **Deny**: Prevents the source service from sending requests to the destination. + 1. **Application Aware**: Enables you to specify L7 criteria for dynamically enforcing intentions. Refer to [Configure application aware settings](#configure-application-aware-settings) for additional information. +1. Click **Save**. + +Repeat the procedure as necessary to create additional intentions. + +#### Configure application aware settings + +You can use the Consul UI to configure L7 permissions. + +1. Click **Add permission** to open the permission editor. +1. Enable the **Allow** or **Deny** option. +1. You can specify a path, request method, and request headers to match. All criteria must be satisfied for Consul to enforce the permission. Refer to the [`Sources[].Permissions[]` parameter](/consul/docs/connect/config-entries/service-intentions#sources-permissions) in the service intentions configuration entry reference for information about the available configuration fields. +1. Click **Save**. + +Repeat the procedure as necessary to create additional permissions. + +## Create multiple intentions + +You can create a service intentions configuration entry to specify default intentions for your service mesh. You can specify default settings for L4 or L7 application-aware traffic. + +### Define a service intention configuration entry + +Configure the following fields: + + + + + +- [`Kind`](/consul/docs/connect/config-entries/service-intentions#kind): Declares the type of configuration entry. Must be set to `service-intentions`. +- [`Name`](/consul/docs/connect/config-entries/service-intentions#kind): Specifies the name of the destination service for intentions defined in the configuration entry. You can use a wildcard character (*) to set L4 intentions for all services that are not protected by specific intentions. Wildcards are not supported for L7 intentions. +- [`Sources`](/consul/docs/connect/config-entries/service-intentions#sources): Specifies an unordered list of all intention sources and the authorizations granted to those sources. Consul stores and evaluates the list in reverse order sorted by intention precedence. +- [`Sources.Action`](/consul/docs/connect/config-entries/service-intentions#sources-action) or [`Sources.Permissions`](/consul/docs/connect/config-entries/service-intentions#sources-permissions): For L4 intentions, set the `Action` field to "allow" or "deny" so that Consul can enforce intentions that match the source service. For L7 intentions, configure the `Permissions` settings, which define a set of application-aware attributes for dynamically matching incoming requests. The `Actions` and `Permissions` settings are mutually exclusive. + + + + + +- [`apiVersion`](/consul/docs/connect/config-entries/service-intentions#apiversion): Specifies the Consul API version. Must be set to `consul.hashicorp.com/v1alpha1`. +- [`kind`](/consul/docs/connect/config-entries/service-intentions#kind): Declares the type of configuration entry. Must be set to `ServiceIntentions`. +- [`spec.destination.name`](/consul/docs/connect/config-entries/service-intentions#spec-destination-name): Specifies the name of the destination service for intentions defined in the configuration entry. You can use a wildcard character (*) to set L4 intentions for all services that are not protected by specific intentions. Wildcards are not supported for L7 intentions. +- [`spec.sources`](/consul/docs/connect/config-entries/service-intentions#spec-sources): Specifies an unordered list of all intention sources and the authorizations granted to those sources. Consul stores and evaluates the list in reverse order sorted by intention precedence. +- [`spec.sources.action`](/consul/docs/connect/config-entries/service-intentions#spec-sources-action) or [`spec.sources.permissions`](/consul/docs/connect/config-entries/service-intentions#spec-sources-permissions): For L4 intentions, set the `action` field to "allow" or "deny" so that Consul can enforce intentions that match the source service. For L7 intentions, configure the `permissions` settings, which define a set of application-aware attributes for dynamically matching incoming requests. The `actions` and `permissions` settings are mutually exclusive. + + + + + +Refer to the [service intentions configuration entry](/consul/docs/connect/config-entries/service-intentions) reference documentation for details about all configuration options. + +Refer to the [example service intentions configurations](/consul/docs/connect/config-entries/service-intentions#examples) for additional guidance. + +#### Interaction with other configuration entries + +L7 intentions defined in a configuration entry are restricted to destination services +configured with an HTTP-based protocol as defined in a corresponding +[service defaults configuration entry](/consul/docs/connect/config-entries/service-defaults) +or globally in a [proxy defaults configuration entry](/consul/docs/connect/config-entries/proxy-defaults). + +### Apply the service intentions configuration entry + +You can apply the configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). In Kubernetes environments, apply the `ServiceIntentions` custom resource definitions (CRD) to implement and manage Consul configuration entries. + +Refer to the following topics for details about applying configuration entries: + +- [How to Use Configuration Entries](/consul/docs/agent/config-entries) +- [Custom Resource Definitions for Consul on Kubernetes](/consul/docs/k8s/crds) \ No newline at end of file diff --git a/website/content/docs/connect/intentions/index.mdx b/website/content/docs/connect/intentions/index.mdx new file mode 100644 index 000000000000..301344c64ec0 --- /dev/null +++ b/website/content/docs/connect/intentions/index.mdx @@ -0,0 +1,91 @@ +--- +layout: docs +page_title: Service mesh intentions overview +description: >- + Intentions are access controls that allow or deny incoming requests to services in the mesh. +--- + +# Service intentions overview + +This topic provides overview information about Consul intentions, which are mechanisms that control traffic communication between services in the Consul service mesh. + +![Diagram showing how service intentions control access between services](/img/consul-connect/consul-service-mesh-intentions-overview.svg) + +## Intention types + +Intentions control traffic communication between services at the network layer, also called _L4_ traffic, or the application layer, also called _L7 traffic_. The protocol that the destination service uses to send and receive traffic determines the type of authorization the intention can enforce. + +### L4 traffic intentions + +If the destination service uses TCP or any non-HTTP-based protocol, then intentions can control traffic based on identities encoded in mTLS certificates. Refer to [Mutual transport layer security (mTLS)](/consul/docs/connect/connect-internals#mutual-transport-layer-security-mtls) for additional information. + +This implementation allows broad all-or-nothing access control between pairs of services. The only requirement is that the service is aware of the TLS handshake that wraps the opaque TCP connection. + +### L7 traffic intentions + +If the destination service uses an HTTP-based protocol, then intentions can enforce access based on application-aware request attributes, in addition to identity-based enforcement, to control traffic between services. Refer to [Service intentions configuration reference](/consul/docs/connect/config-entries/service-intentions#permissions) for additional information. + +## Workflow + +You can manually create intentions from the Consul UI, API, or CLI. You can also enable Consul to dynamically create them by defining traffic routes in service intention configuration entries. Refer to [Create and manage intentions](/consul/docs/connect/intentions/create-manage-intentions) for details. + +### Enforcement + +The [proxy](/consul/docs/connect/proxies) or [natively-integrated +application](/consul/docs/connect/native) enforces intentions on inbound connections or requests. Only one intention can control authorization between a pair of services at any single point in time. + +L4 intentions mediate the ability to establish new connections. Modifying an intention does not have an effect on existing connections. As a result, changing a connection from `allow` to `deny` does not sever the connection. + +L7 intentions mediate the ability to issue new requests. When an intention is modified, requests received after the modification use the latest intention rules to enforce access. Changing a connection from `allow` to `deny` does not sever the connection, but doing so blocks new requests from being processed. + +### Caching + +The intentions for services registered with a Consul agent are cached locally on the agent. Supported proxies also cache intention data in their own configurations so that they can authorize inbound connections or requests without relying on the Consul agent. All actions in the data path of connections take place within the proxy. + +### Updates + +Consul propagates updates to intentions almost instantly as a result of the continuous blocking query the agent uses. A _blocking query_ is a Consul API feature that uses long polling to wait for potential changes. Refer to [Blocking Queries](/consul/api-docs/features/blocking) for additional information. Proxies also use blocking queries to quickly update their local configurations. + +Because all intention data is cached locally, authorizations for inbound connection persist, even if the agents are completely severed from the Consul servers or if the proxies are completely severed from their local Consul agent. If the connection is severed, Consul automatically applies changes to intentions when connectivity is restored. + +### Intention maintenance + +Services should periodically call the [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) to retrieve all relevant intentions for the target destination. After verifying the TLS client certificate, the cached intentions for each incoming connection or request determine if it should be accepted or rejected. + +## Precedence and match order + +Consul processes criteria defined in the service intention configuration entry to match incoming requests. When Consul finds a match, it applies the corresponding action specified in the configuration entry. The match criteria may include specific HTTP headers, request methods, or other attributes. Additionally, you can use regular expressions to programmatically match attributes. Refer to [Service intention configuration entry reference](/consul/docs/connect/config-entries/service-intentions) for details. + +Consul orders the matches based the following factors: + +- Specificity: Incoming requests that match attributes directly have the highest precedence. For example, intentions that are configured to deny traffic from services that send `POST` requests take precedence over intentions that allow traffic from methods configured with the wildcard value `*`. +- Authorization: Consul enforces `deny` over `allow` if match criteria are weighted equally. + +The following table shows match precedence in descending order: + +| Precedence | Source Namespace | Source Name | Destination Namespace | Destination Name | +| -----------| ---------------- | ------------| --------------------- | ---------------- | +| 9 | Exact | Exact | Exact | Exact | +| 8 | Exact | `*` | Exact | Exact | +| 7 | `*` | `*` | Exact | Exact | +| 6 | Exact | Exact | Exact | `*` | +| 5 | Exact | `*` | Exact | `*` | +| 4 | `*` | `*` | Exact | `*` | +| 3 | Exact | Exact | `*` | `*` | +| 2 | Exact | `*` | `*` | `*` | +| 1 | `*` | `*` | `*` | `*` | + +Consul prints the precedence value to the service intentions configuration entry after it processes the matching criteria. The value is read-only. Refer to +[`Precedence`](/consul/docs/connect/config-entries/service-intentions#precedence) for additional information. + +Namespaces are an Enterprise feature. In Consul OSS, the only allowable value for either namespace field is `"default"`. Other rows in the table are not applicable. + +The [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) +should be periodically called to retrieve all relevant intentions for the +target destination. After verifying the TLS client certificate, the cached +intentions should be consulted for each incoming connection/request to +determine if it should be accepted or rejected. + +The default intention behavior is defined by the [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. +If the configuration is set `allow`, then all service mesh Connect connections will be allowed by default. +If is set to `deny`, then all connections or requests will be denied by default. \ No newline at end of file diff --git a/website/content/docs/connect/intentions-legacy.mdx b/website/content/docs/connect/intentions/legacy.mdx similarity index 98% rename from website/content/docs/connect/intentions-legacy.mdx rename to website/content/docs/connect/intentions/legacy.mdx index 0e9e991d37c5..b79d34a543ae 100644 --- a/website/content/docs/connect/intentions-legacy.mdx +++ b/website/content/docs/connect/intentions/legacy.mdx @@ -8,8 +8,7 @@ description: >- # Intentions in Legacy Mode ~> **1.8.x and earlier:** This document only applies in Consul versions 1.8.x -and before. If you are using version 1.9.0 or later please use the updated -documentation [here](/consul/docs/connect/intentions). +and before. If you are using version 1.9.0 or later, refer to the [current intentions documentation](/consul/docs/connect/intentions). Intentions define access control for services via Connect and are used to control which services may establish connections. Intentions can be diff --git a/website/content/docs/k8s/connect/index.mdx b/website/content/docs/k8s/connect/index.mdx index 7e30ab2be898..00e5c8e9383e 100644 --- a/website/content/docs/k8s/connect/index.mdx +++ b/website/content/docs/k8s/connect/index.mdx @@ -175,18 +175,18 @@ upstream. This is analogous to the standard Kubernetes service environment varia point instead to the correct local proxy port to establish connections via Connect. -We can verify access to the static text server using `kubectl exec`. +You can verify access to the static text server using `kubectl exec`. Because transparent proxy is enabled by default, -we use Kubernetes DNS to connect to our desired upstream. +use Kubernetes DNS to connect to your desired upstream. ```shell-session $ kubectl exec deploy/static-client -- curl --silent http://static-server/ "hello world" ``` -We can control access to the server using [intentions](/consul/docs/connect/intentions). +You can control access to the server using [intentions](/consul/docs/connect/intentions). If you use the Consul UI or [CLI](/consul/commands/intention/create) to -create a deny [intention](/consul/docs/connect/intentions) between +deny communication between "static-client" and "static-server", connections are immediately rejected without updating either of the running pods. You can then remove this intention to allow connections again. diff --git a/website/content/docs/lambda/invoke-from-lambda.mdx b/website/content/docs/lambda/invoke-from-lambda.mdx index a7ec2913865c..4c29e4b8d9cd 100644 --- a/website/content/docs/lambda/invoke-from-lambda.mdx +++ b/website/content/docs/lambda/invoke-from-lambda.mdx @@ -264,7 +264,7 @@ Define the following environment variables in your Lambda functions to configure ## Invoke the Lambda function -If _intentions_ are enabled in the Consul service mesh, you must create an intention that allows the Lambda function's Consul service to invoke all upstream services prior to invoking the Lambda function. Refer to [Service Mesh Intentions](/consul/docs/connect/intentions) for additional information. +If _intentions_ are enabled in the Consul service mesh, you must create an intention that allows the Lambda function's Consul service to invoke all upstream services prior to invoking the Lambda function. Refer to [Service mesh intentions](/consul/docs/connect/intentions) for additional information. There are several ways to invoke Lambda functions. In the following example, the `aws lambda invoke` CLI command invokes the function: diff --git a/website/content/docs/security/acl/acl-rules.mdx b/website/content/docs/security/acl/acl-rules.mdx index 3b63e327a20e..a09ffd2fc253 100644 --- a/website/content/docs/security/acl/acl-rules.mdx +++ b/website/content/docs/security/acl/acl-rules.mdx @@ -869,7 +869,7 @@ service "app" { -Refer to [Intention Management Permissions](/consul/docs/connect/intentions#intention-management-permissions) +Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for more information about managing intentions access with service rules. ## Session Rules diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index f66afc94c5f1..67317ed5fa77 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -539,12 +539,25 @@ ] }, { - "title": "Service-to-service permissions - Intentions", - "path": "connect/intentions" - }, - { - "title": "Service-to-service permissions - Intentions (Legacy Mode)", - "path": "connect/intentions-legacy" + "title": "Service intentions", + "routes": [ + { + "title": "Overview", + "path": "connect/intentions" + }, + { + "title": "Create and manage service intentions", + "path": "connect/intentions/create-manage-intentions" + }, + { + "title": "Service intentions legacy mode", + "path": "connect/intentions/legacy" + }, + { + "title": "Configuration", + "href": "/consul/docs/connect/config-entries/service-intentions" + } + ] }, { "title": "Transparent Proxy", diff --git a/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg b/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg new file mode 100644 index 000000000000..cdd7ea0ad6eb --- /dev/null +++ b/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg @@ -0,0 +1 @@ + diff --git a/website/redirects.js b/website/redirects.js index 3e4625016505..fb9ec442c664 100644 --- a/website/redirects.js +++ b/website/redirects.js @@ -22,4 +22,14 @@ module.exports = [ destination: '/consul/docs/k8s/connect/cluster-peering/tech-specs', permanent: true, }, + { + source: '/consul/docs/connect/intentions#intention-management-permissions', + destination: `/consul/docs/connect/intentions/create-manage-intentions#acl-requirements`, + permanent: true, + }, + { + source: '/consul/docs/connect/intentions#intention-basics', + destination: `/consul/docs/connect/intentions`, + permanent: true, + }, ] From da0b43a4876f5e198384615eb3c217916da200e8 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Mon, 27 Mar 2023 13:02:44 -0400 Subject: [PATCH 130/421] Fix bug in changelog checker where bash variable is not quoted (#16681) (#16785) --- .github/scripts/changelog_checker.sh | 23 +++++++++++++++++++++ .github/workflows/changelog-checker.yml | 27 ++++++------------------- 2 files changed, 29 insertions(+), 21 deletions(-) create mode 100755 .github/scripts/changelog_checker.sh diff --git a/.github/scripts/changelog_checker.sh b/.github/scripts/changelog_checker.sh new file mode 100755 index 000000000000..e46030da1e16 --- /dev/null +++ b/.github/scripts/changelog_checker.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +# check if there is a diff in the .changelog directory +# for PRs against the main branch, the changelog file name should match the PR number +if [ "$GITHUB_BASE_REF" = "$GITHUB_DEFAULT_BRANCH" ]; then + enforce_matching_pull_request_number="matching this PR number " + changelog_file_path=".changelog/(_)?$PR_NUMBER.txt" +else + changelog_file_path=".changelog/[_0-9]*.txt" +fi + +changelog_files=$(git --no-pager diff --name-only HEAD "$(git merge-base HEAD "origin/main")" | egrep "${changelog_file_path}") + +# If we do not find a file in .changelog/, we fail the check +if [ -z "$changelog_files" ]; then + # Fail status check when no .changelog entry was found on the PR + echo "Did not find a .changelog entry ${enforce_matching_pull_request_number}and the 'pr/no-changelog' label was not applied. Reference - https://github.com/hashicorp/consul/pull/8387" + exit 1 +else + echo "Found .changelog entry in PR!" +fi diff --git a/.github/workflows/changelog-checker.yml b/.github/workflows/changelog-checker.yml index bb13255d297e..c81eb8a7a041 100644 --- a/.github/workflows/changelog-checker.yml +++ b/.github/workflows/changelog-checker.yml @@ -15,7 +15,7 @@ jobs: # checks that a .changelog entry is present for a PR changelog-check: # If there a `pr/no-changelog` label we ignore this check. Also, we ignore PRs created by the bot assigned to `backport-assistant` - if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-changelog') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" + if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-changelog') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" runs-on: ubuntu-latest steps: @@ -24,23 +24,8 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 # by default the checkout action doesn't checkout all branches - name: Check for changelog entry in diff - run: | - # check if there is a diff in the .changelog directory - # for PRs against the main branch, the changelog file name should match the PR number - if [ "${{ github.event.pull_request.base.ref }}" = "${{ github.event.repository.default_branch }}" ]; then - enforce_matching_pull_request_number="matching this PR number " - changelog_file_path=".changelog/(_)?${{ github.event.pull_request.number }}.txt" - else - changelog_file_path=".changelog/[_0-9]*.txt" - fi - - changelog_files=$(git --no-pager diff --name-only HEAD "$(git merge-base HEAD "origin/main")" | egrep ${changelog_file_path}) - - # If we do not find a file in .changelog/, we fail the check - if [ -z "$changelog_files" ]; then - # Fail status check when no .changelog entry was found on the PR - echo "Did not find a .changelog entry ${enforce_matching_pull_request_number}and the 'pr/no-changelog' label was not applied. Reference - https://github.com/hashicorp/consul/pull/8387" - exit 1 - else - echo "Found .changelog entry in PR!" - fi + run: ./.github/scripts/changelog_checker.sh + env: + GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref }} + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + PR_NUMBER: ${{ github.event.pull_request.number }} From 1553036635b12347d4c7b439818a22be72ff450a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 27 Mar 2023 11:54:00 -0700 Subject: [PATCH 131/421] Backport of Fix struct tags for TCPService enterprise meta into release/1.15.x (#16787) * backport of commit de17e5a23df471d3741719082f3d31b599389c2c * backport of commit 85480c053d8b33b0244d1d7b8ca0908d0cf12176 --------- Co-authored-by: jm96441n --- .changelog/16781.txt | 3 +++ agent/structs/config_entry_routes.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changelog/16781.txt diff --git a/.changelog/16781.txt b/.changelog/16781.txt new file mode 100644 index 000000000000..708a91d40c86 --- /dev/null +++ b/.changelog/16781.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal for TCPServices. +``` diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 683ec9f3fac0..70a306559880 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -556,7 +556,7 @@ func (e *TCPRouteConfigEntry) CanWrite(authz acl.Authorizer) error { type TCPService struct { Name string - acl.EnterpriseMeta + acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` } func (s TCPService) ServiceName() ServiceName { From 7d44a4c2f1b1358f78790096481957cab4ea2657 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Mar 2023 09:58:25 -0700 Subject: [PATCH 132/421] Backport of Update normalization of route refs into release/1.15.x (#16799) * backport of commit 7f5e9ba4cde01e5e2037eafc80a5b22eb0124791 * backport of commit 67b85371e0a06d36d8ad18dc4678e1216ee52012 * backport of commit b47040d1edda4ce724cdd6161acf5e9779a9836f * backport of commit 079ed984c0a6833b7228179ace42c8aa5b4c3106 * Fix infinite call loop * Explicitly call enterprise meta --------- Co-authored-by: jm96441n --- .changelog/16789.txt | 3 +++ agent/structs/config_entry_gateways.go | 3 +++ agent/structs/config_entry_routes.go | 8 ++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changelog/16789.txt diff --git a/.changelog/16789.txt b/.changelog/16789.txt new file mode 100644 index 000000000000..ed25e11bbb64 --- /dev/null +++ b/.changelog/16789.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where parent refs and service refs for a route in the same namespace as the route would fallback to the default namespace if the namespace was not specified in the configuration rather than falling back to the routes namespace. +``` diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 5309af35ad3d..32fd966f6000 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -755,6 +755,7 @@ func (e *APIGatewayConfigEntry) Normalize() error { if cert.Kind == "" { cert.Kind = InlineCertificate } + cert.EnterpriseMeta.Merge(e.GetEnterpriseMeta()) cert.EnterpriseMeta.Normalize() listener.TLS.Certificates[i] = cert @@ -985,11 +986,13 @@ func (e *BoundAPIGatewayConfigEntry) GetMeta() map[string]string { return e.Meta func (e *BoundAPIGatewayConfigEntry) Normalize() error { for i, listener := range e.Listeners { for j, route := range listener.Routes { + route.EnterpriseMeta.Merge(&e.EnterpriseMeta) route.EnterpriseMeta.Normalize() listener.Routes[j] = route } for j, cert := range listener.Certificates { + cert.EnterpriseMeta.Merge(&e.EnterpriseMeta) cert.EnterpriseMeta.Normalize() listener.Certificates[j] = cert diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index 70a306559880..ca092c07d196 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -81,6 +81,7 @@ func (e *HTTPRouteConfigEntry) Normalize() error { if parent.Kind == "" { parent.Kind = APIGateway } + parent.EnterpriseMeta.Merge(e.GetEnterpriseMeta()) parent.EnterpriseMeta.Normalize() e.Parents[i] = parent } @@ -91,7 +92,7 @@ func (e *HTTPRouteConfigEntry) Normalize() error { } for j, service := range rule.Services { - rule.Services[j] = normalizeHTTPService(service) + rule.Services[j] = e.normalizeHTTPService(service) } e.Rules[i] = rule } @@ -99,7 +100,8 @@ func (e *HTTPRouteConfigEntry) Normalize() error { return nil } -func normalizeHTTPService(service HTTPService) HTTPService { +func (e *HTTPRouteConfigEntry) normalizeHTTPService(service HTTPService) HTTPService { + service.EnterpriseMeta.Merge(e.GetEnterpriseMeta()) service.EnterpriseMeta.Normalize() if service.Weight <= 0 { service.Weight = 1 @@ -507,11 +509,13 @@ func (e *TCPRouteConfigEntry) Normalize() error { if parent.Kind == "" { parent.Kind = APIGateway } + parent.EnterpriseMeta.Merge(e.GetEnterpriseMeta()) parent.EnterpriseMeta.Normalize() e.Parents[i] = parent } for i, service := range e.Services { + service.EnterpriseMeta.Merge(e.GetEnterpriseMeta()) service.EnterpriseMeta.Normalize() e.Services[i] = service } From aeee58bd6044f6d414726e3decf369025a31079f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 28 Mar 2023 15:52:51 -0700 Subject: [PATCH 133/421] Backport of Github Actions Migration - move go-tests workflows to GHA into release/1.15.x (#16803) * backport of commit df9d97f9943e9bf0a32ae618098917bd71606773 * backport of commit 8913844f1da6247347c4a41b133ef78ca56d8afb * backport of commit e8d708b6931edb6fa4d35e9cae5f9fa0b617d536 * backport of commit b42be82b322db542a25d3b0007dac663cc983da1 * backport of commit 2e7fc805c974fab9f34c09ea7403a38bb9c15e1c * backport of commit 6e65358f2a6a46bd1a6a954b1205170a01c096eb * backport of commit 924060ef91eab4fe23186e7abdd504b0e7b48457 * backport of commit b4f5218a29caa71ad5e6e0472bcf0120a3efe22b * backport of commit 4a3c1a2b5279b66a59612ee708f0cf626de4d751 * backport of commit e6f0d8e361ca43920e3de6309ee9edd93f682f70 * backport of commit 9eed445d5021c63882eee13da49c8a4177fc7da4 * backport of commit 848170652d7a413ab56e145a46fd657deac726c7 * backport of commit 5aab67566395fc39aded52bed8bbda3477ec86a1 * backport of commit 07d8ede1ff363be46c29b4f4e07291f2cc57d027 * backport of commit 5b1b1acb82df179d18a360131b7b6873d5c0ada9 * backport of commit 2ca46f812daccca2bf4f32802f06bf23218de708 * backport of commit c9f85388b1fd4d6ab03ee000d421e6a68ffb674f * backport of commit c450fa7d43da5a16b4f8dd2e3f1f0d23d444b844 * backport of commit 9302ebc9cff3f42ae71b3229216e63df67922fe6 * backport of commit 0f55035288595d5d5e6ceb8cc9e88d845f3001af * backport of commit 76694ab441022e85c9bab7b1d46ea45e732673f8 * backport of commit e7a7d983a5d2f790cf55ac9986fd16ee19775fec * backport of commit fea35f520370b45aafdfd19147e0e0af67427c2b * backport of commit d64af3f727563464ea912abccc96d61f8ab46043 * backport of commit 840d80a1785cd7ad03816c74d76bdade8a0811f3 --------- Co-authored-by: John Murret Co-authored-by: Dan Bond --- .github/scripts/get_runner_classes.sh | 22 ++ .github/scripts/notify_slack.sh | 27 ++ .github/scripts/rerun_fails_report.sh | 24 ++ .github/scripts/set_test_package_matrix.sh | 8 + .github/workflows/build-distros.yml | 48 ++-- .github/workflows/frontend.yml | 94 +++++++ .github/workflows/go-tests.yml | 267 ++++++++++++++++++++ .github/workflows/reusable-check-go-mod.yml | 28 ++ .github/workflows/reusable-dev-build.yml | 31 +++ .github/workflows/reusable-lint.yml | 47 ++++ .github/workflows/reusable-unit-split.yml | 128 ++++++++++ .github/workflows/reusable-unit.yml | 107 ++++++++ .github/workflows/verify-ci.yml | 17 ++ 13 files changed, 830 insertions(+), 18 deletions(-) create mode 100755 .github/scripts/get_runner_classes.sh create mode 100755 .github/scripts/notify_slack.sh create mode 100755 .github/scripts/rerun_fails_report.sh create mode 100755 .github/scripts/set_test_package_matrix.sh create mode 100644 .github/workflows/frontend.yml create mode 100644 .github/workflows/go-tests.yml create mode 100644 .github/workflows/reusable-check-go-mod.yml create mode 100644 .github/workflows/reusable-dev-build.yml create mode 100644 .github/workflows/reusable-lint.yml create mode 100644 .github/workflows/reusable-unit-split.yml create mode 100644 .github/workflows/reusable-unit.yml create mode 100644 .github/workflows/verify-ci.yml diff --git a/.github/scripts/get_runner_classes.sh b/.github/scripts/get_runner_classes.sh new file mode 100755 index 000000000000..4834e9e92917 --- /dev/null +++ b/.github/scripts/get_runner_classes.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# This script generates tag-sets that can be used as runs-on: values to select runners. + +set -euo pipefail + +case "$GITHUB_REPOSITORY" in + *-enterprise) + # shellcheck disable=SC2129 + echo "compute-small=['self-hosted', 'linux', 'small']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['self-hosted', 'linux', 'medium']" >> "$GITHUB_OUTPUT" + echo "compute-large=['self-hosted', 'linux', 'large']" >> "$GITHUB_OUTPUT" + echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5.2xlarge']" >> "$GITHUB_OUTPUT" + ;; + *) + # shellcheck disable=SC2129 + echo "compute-small=['custom-linux-s-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['custom-linux-m-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-large=['custom-linux-l-consul-latest']" >> "$GITHUB_OUTPUT" + echo "compute-xl=['custom-linux-xl-consul-latest']" >> "$GITHUB_OUTPUT" + ;; +esac diff --git a/.github/scripts/notify_slack.sh b/.github/scripts/notify_slack.sh new file mode 100755 index 000000000000..b3dcdb210dc6 --- /dev/null +++ b/.github/scripts/notify_slack.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -uo pipefail + +# This script is used in GitHub Actions pipelines to notify Slack of a job failure. + +if [[ $GITHUB_REF_NAME == "main" ]]; then + GITHUB_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" + GITHUB_ACTIONS_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1) + SHORT_REF=$(git rev-parse --short "${GITHUB_SHA}") + curl -X POST -H 'Content-type: application/json' \ + --data \ + "{ \ + \"attachments\": [ \ + { \ + \"fallback\": \"GitHub Actions workflow failed!\", \ + \"text\": \"❌ Failed: \`${GITHUB_ACTOR}\`'s <${GITHUB_ACTIONS_ENDPOINT}|${GITHUB_JOB}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${GITHUB_REF_NAME}\`!\n\n- <${COMMIT_MESSAGE}\", \ + \"footer\": \"${GITHUB_REPOSITORY}\", \ + \"ts\": \"$(date +%s)\", \ + \"color\": \"danger\" \ + } \ + ] \ + }" "${FEED_CONSUL_GH_URL}" +else + echo "Not posting slack failure notifications for non-main branch" +fi diff --git a/.github/scripts/rerun_fails_report.sh b/.github/scripts/rerun_fails_report.sh new file mode 100755 index 000000000000..ac6b7cf2ff9d --- /dev/null +++ b/.github/scripts/rerun_fails_report.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# +# Add a comment on the github PR if there were any rerun tests. +# +set -eu -o pipefail + +report_filename="${1?report filename is required}" +if [ ! -s "$report_filename" ]; then + echo "gotestsum rerun report file is empty or missing" + exit 0 +fi + +function report { + echo ":repeat: gotestsum re-ran some tests in https://github.com/hashicorp/consul/actions/run/$GITHUB_RUN_ID" + echo + echo '```' + cat "$report_filename" + echo '```' +} + +report diff --git a/.github/scripts/set_test_package_matrix.sh b/.github/scripts/set_test_package_matrix.sh new file mode 100755 index 000000000000..a9a21c747e57 --- /dev/null +++ b/.github/scripts/set_test_package_matrix.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +export RUNNER_COUNT=$1 + +# set matrix var to list of unique packages containing tests +matrix="$(go list -json="ImportPath,TestGoFiles" ./... | jq --compact-output '. | select(.TestGoFiles != null) | .ImportPath' | jq --slurp --compact-output '.' | jq --argjson runnercount $RUNNER_COUNT -cM '[_nwise(length / $runnercount | floor)]'))" + +echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 87d6f9605442..620504e4e4d1 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -8,26 +8,34 @@ permissions: contents: read jobs: - check-go-mod: - runs-on: ubuntu-22.04 + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 - with: - go-version-file: 'go.mod' - - run: go mod tidy - - run: | - if [[ -n $(git status -s) ]]; then - echo "Git directory has changes" - git status -s - exit 1 - fi + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-medium }} build-386: - needs: check-go-mod + needs: + - setup + - check-go-mod env: XC_OS: "freebsd linux windows" - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 @@ -40,10 +48,12 @@ jobs: done build-amd64: - needs: check-go-mod + needs: + - setup + - check-go-mod env: XC_OS: "darwin freebsd linux solaris windows" - runs-on: ubuntu-22.04 + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 @@ -56,8 +66,10 @@ jobs: done build-arm: - needs: check-go-mod - runs-on: ubuntu-22.04 + needs: + - setup + - check-go-mod + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} env: CGO_ENABLED: 1 GOOS: linux diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000000..e7d9e385514a --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,94 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: frontend + +on: + push: + branches: + - main + - ui/** + - backport/ui/** + +permissions: + contents: read + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + workspace-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-workspace + + node-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-node + working-directory: ui/packages/consul-ui + + ember-build-test: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + env: + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam + CONSUL_NSPACES_ENABLED: 0 # NOTE: this should be 1 in ENT. + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - working-directory: ui/packages/consul-ui + run: | + make build-ci + node_modules/.bin/ember exam --path dist --silent -r xunit + + - working-directory: ui/packages/consul-ui + run: make test-coverage-ci diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml new file mode 100644 index 000000000000..9c799fa9a4cb --- /dev/null +++ b/.github/workflows/go-tests.yml @@ -0,0 +1,267 @@ +name: go-tests + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branches' prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +permissions: + contents: read + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-small }} + + check-generated-protobuf: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make proto-tools + name: Install protobuf + - run: make proto-format + name: "Protobuf Format" + - run: make --always-make proto + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - run: make proto-lint + name: "Protobuf Lint" + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + check-generated-deep-copy: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make codegen-tools + name: Install deep-copy + - run: make --always-make deep-copy + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + lint-enums: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint-container-test-deps: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: make lint-container-test-deps + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint-consul-retry: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh + + lint: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + + lint-32bit: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + go-arch: "386" + runs-on: ${{ needs.setup.outputs.compute-xl }} + + + # create a development build + dev-build: + needs: + - setup + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + + # TODO(JM): - linux arm64 is not available in our self-hosted runners + # they are currently on the roadmap. + # # create a development build for arm64 + # dev-build-arm64: + # needs: + # - setup + # uses: ./.github/workflows/reusable-dev-build.yml + # with: + # uploaded-binary-name: 'consul-bin-arm64' + # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} + + # go-test-arm64: + # # TODO(JM): Fix to run on arm64 + # needs: + # - setup + # - dev-build-arm64 + # uses: ./.github/workflows/reusable-unit-split.yml + # with: + # directory: . + # uploaded-binary-name: 'consul-bin-arm64' + # runner-count: 12 + # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} + # go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' + + go-test: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + runner-count: 12 + runs-on: ${{ needs.setup.outputs.compute-xl }} + + + go-test-race: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"' + package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul/command/'" + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-32bit: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-arch: "386" + go-test-flags: 'export GO_TEST_FLAGS="-short"' + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-envoyextensions: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: envoyextensions + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-troubleshoot: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: troubleshoot + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-api-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-api-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-sdk-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-xl }} + + go-test-sdk-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-xl }} + + noop: + runs-on: ubuntu-latest + steps: + - run: "echo ok" diff --git a/.github/workflows/reusable-check-go-mod.yml b/.github/workflows/reusable-check-go-mod.yml new file mode 100644 index 000000000000..ab6ef2420d6a --- /dev/null +++ b/.github/workflows/reusable-check-go-mod.yml @@ -0,0 +1,28 @@ +name: check-go-mod + +on: + workflow_call: + inputs: + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string +jobs: + check-go-mod: + runs-on: ${{ fromJSON(inputs.runs-on) }} + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go mod tidy + - run: | + if [[ -n $(git status -s) ]]; then + echo "Git directory has changes" + git status -s + exit 1 + fi + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml new file mode 100644 index 000000000000..d4832fb0a98f --- /dev/null +++ b/.github/workflows/reusable-dev-build.yml @@ -0,0 +1,31 @@ +name: reusable-dev-build + +on: + workflow_call: + inputs: + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string +jobs: + build: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Build + run: make dev + # save dev build to pass to downstream jobs + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: ./bin/consul + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-lint.yml b/.github/workflows/reusable-lint.yml new file mode 100644 index 000000000000..121d9231fa63 --- /dev/null +++ b/.github/workflows/reusable-lint.yml @@ -0,0 +1,47 @@ +name: reusable-lint + +on: + workflow_call: + inputs: + go-arch: + required: false + type: string + default: "" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + +env: + GOTAGS: "" # No tags for OSS but there are for enterprise + GOARCH: ${{inputs.go-arch}} + +jobs: + lint: + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + matrix: + directory: + - "" + - "api" + - "sdk" + - "envoyextensions" + - "troubleshoot" + - "test/integration/consul-container" + fail-fast: true + name: lint ${{ matrix.directory }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + - name: lint-${{ matrix.directory }} + uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # pin@v3.4.0 + with: + working-directory: ${{ matrix.directory }} + version: v1.51 + args: --build-tags="${{ env.GOTAGS }}" -v + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml new file mode 100644 index 000000000000..fbd4d859bf52 --- /dev/null +++ b/.github/workflows/reusable-unit-split.yml @@ -0,0 +1,128 @@ +name: reusable-unit-split + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + args: + required: false + type: string + default: "" + runner-count: + required: false + type: number + default: 1 + go-test-flags: + required: false + type: string + default: "" + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + GOARCH: ${{inputs.go-arch}} + TOTAL_RUNNERS: ${{inputs.runner-count}} + +jobs: + set-test-package-matrix: + runs-on: ubuntu-latest + outputs: + package-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - id: set-matrix + run: ./.github/scripts/set_test_package_matrix.sh ${{env.TOTAL_RUNNERS}} + + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + name: "go-test" + needs: + - set-test-package-matrix + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.set-test-package-matrix.outputs.package-matrix) }} + steps: + - name: ulimit + run: | + echo "Soft limits" + ulimit -Sa + echo "Hard limits" + ulimit -Ha + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + cache: true + - name: Install gotestsum + run: | + wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: /usr/local/bin + - name: Make sure consul is executable + run: sudo chmod +x /usr/local/bin/consul + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + # separate the list + PACKAGE_NAMES="${{ join(matrix.package, ' ') }}" + # PACKAGE_NAMES="${{ matrix.package }}" + + ${{inputs.go-test-flags}} + + # some tests expect this umask, and arm images have a different default + umask 0022 + + gotestsum \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" -p 2 \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml new file mode 100644 index 000000000000..24c20abe52d4 --- /dev/null +++ b/.github/workflows/reusable-unit.yml @@ -0,0 +1,107 @@ +name: reusable-unit + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + package-names-command: + required: false + type: string + default: 'go list -tags "$GOTAGS" ./...' + go-test-flags: + required: false + type: string + default: "" + +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: 1.8.2 + GOARCH: ${{inputs.go-arch}} + +jobs: + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - name: Setup go mod cache + uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # pin@v3.2.6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Install gotestsum + run: | + wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: /usr/local/bin + - name: Make sure consul is executable + run: sudo chmod +x /usr/local/bin/consul + - name: Display downloaded file + run: ls -ld consul + working-directory: /usr/local/bin + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + PACKAGE_NAMES=$(${{inputs.package-names-command}}) + + # some tests expect this umask, and arm images have a different default + umask 0022 + + ${{inputs.go-test-flags}} + + gotestsum \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: failure() + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/verify-ci.yml b/.github/workflows/verify-ci.yml new file mode 100644 index 000000000000..16fb4db25be5 --- /dev/null +++ b/.github/workflows/verify-ci.yml @@ -0,0 +1,17 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# verify-ci is a no-op workflow that must run on every PR. It is used in a +# branch protection rule to detect when CI workflows are not running. +name: verify-ci + +permissions: + contents: read + +on: [pull_request] + +jobs: + noop: + runs-on: ubuntu-latest + steps: + - run: echo "ok" From 2425a544761b391db8ed60371635f649fa446e74 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 29 Mar 2023 11:03:04 -0700 Subject: [PATCH 134/421] backport of commit 4b2077fbae54f4f2b48a768fe90633288819b17f (#16812) Co-authored-by: Rosemary Wang <915624+joatmon08@users.noreply.github.com> --- .../content/docs/connect/proxies/envoy-extensions/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/connect/proxies/envoy-extensions/index.mdx b/website/content/docs/connect/proxies/envoy-extensions/index.mdx index 844ab0706c3a..ecd9e63e0740 100644 --- a/website/content/docs/connect/proxies/envoy-extensions/index.mdx +++ b/website/content/docs/connect/proxies/envoy-extensions/index.mdx @@ -24,8 +24,8 @@ Envoy extensions enable additional service mesh functionality in Consul by chang ### Lua -The `lua` Envoy extension enables HTTP Lua filters in your Consul Envoy proxies. It allows you to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. Refer to the [`lua`](/consul/docs/proxies/envoy-extensions/usage/lua) documentation for more information. +The `lua` Envoy extension enables HTTP Lua filters in your Consul Envoy proxies. It allows you to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. Refer to the [`lua`](/consul/docs/connect/proxies/envoy-extensions/usage/lua) documentation for more information. ### Lambda -The `lambda` Envoy extension enables services to make requests to AWS Lambda functions through the mesh as if they are a normal part of the Consul catalog. Refer to the [`lambda`](/consul/docs/proxies/envoy-extensions/usage/lambda) documentation for more information. +The `lambda` Envoy extension enables services to make requests to AWS Lambda functions through the mesh as if they are a normal part of the Consul catalog. Refer to the [`lambda`](/consul/docs/connect/proxies/envoy-extensions/usage/lambda) documentation for more information. From f87afd63ac52716c907508d722930122e32e4183 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 29 Mar 2023 12:47:12 -0700 Subject: [PATCH 135/421] Backport of docs: Updates to support HCP Consul cluster peering release into release/1.15.x (#16809) * backport of commit a04e4762a2b5dccb7dbb17767a733372e896bd8b * backport of commit 05702b5b5928735a49cbfad722ff10a20f62dfdc * docs: Updates to support HCP Consul cluster peering release (#16774) * New HCP Consul documentation section + links * Establish cluster peering usage cross-link * unrelated fix to backport to v1.15 * nav correction + fixes * Tech specs fixes * specifications for headers * Tech specs fixes + alignments * sprawl edits * Tip -> note --------- Co-authored-by: boruszak Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --- .../docs/connect/cluster-peering/index.mdx | 15 +++- .../connect/cluster-peering/tech-specs.mdx | 61 ++++++++----- .../usage/establish-cluster-peering.mdx | 8 +- .../connect/cluster-peering/tech-specs.mdx | 87 ++++++++----------- .../discovery/dns-dynamic-lookups.mdx | 17 +++- website/data/docs-nav-data.json | 4 - 6 files changed, 103 insertions(+), 89 deletions(-) diff --git a/website/content/docs/connect/cluster-peering/index.mdx b/website/content/docs/connect/cluster-peering/index.mdx index f8e1c18caf3a..f69f093330ef 100644 --- a/website/content/docs/connect/cluster-peering/index.mdx +++ b/website/content/docs/connect/cluster-peering/index.mdx @@ -51,24 +51,31 @@ Regardless of whether you connect your clusters through WAN federation or cluste The following resources are available to help you use Consul's cluster peering features. -**Tutorials:** +### Tutorials - To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) and Google Kubernetes Engine (GKE) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering). -**Usage documentation:** +### Usage documentation - [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering) - [Manage cluster peering connections](/consul/docs/connect/cluster-peering/usage/manage-connections) - [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management) -**Kubernetes-specific documentation:** +### Kubernetes documentation - [Cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs) - [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering) - [Manage cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/manage-peering) - [Manage L7 traffic with cluster peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/l7-traffic) -**Reference documentation:** +### HCP Consul documentation + +- [Cluster peering](/hcp/docs/consul/usage/cluster-peering) +- [Cluster peering topologies](/hcp/docs/consul/usage/cluster-peering/topologies) +- [Establish cluster peering connnections on HCP Consul](/hcp/docs/consul/usage/cluster-peering/create-connections) +- [Cluster peering with management plane](/hcp/docs/consul/usage/management-plane#cluster-peering) + +### Reference documentation - [Cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs) - [HTTP API reference: `/peering/` endpoint](/consul/api-docs/peering) diff --git a/website/content/docs/connect/cluster-peering/tech-specs.mdx b/website/content/docs/connect/cluster-peering/tech-specs.mdx index e34c0d190938..2dd335407e6c 100644 --- a/website/content/docs/connect/cluster-peering/tech-specs.mdx +++ b/website/content/docs/connect/cluster-peering/tech-specs.mdx @@ -7,43 +7,60 @@ description: >- # Cluster peering technical specifications -This reference topic describes the technical specifications associated with using cluster peering in your deployments. These specifications include required Consul components and their configurations. +This reference topic describes the technical specifications associated with using cluster peering in your deployments. These specifications include required Consul components and their configurations. To learn more about Consul's cluster peering feature, refer to [cluster peering overview](/consul/docs/connect/cluster-peering). + +For cluster peering requirements in Kubernetes deployments, refer to [cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs). ## Requirements -To use cluster peering features, make sure your Consul environment meets the following prerequisites: +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. + +In addition, make sure your Consul environment meets the following prerequisites: - Consul v1.14 or higher. -- A local Consul agent is required to manage mesh gateway configuration. - Use [Envoy proxies](/consul/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. +- A local Consul agent is required to manage mesh gateway configurations. -In addition, the following service mesh components are required in order to establish cluster peering connections: - -- [Mesh gateways](#mesh-gateway-requirements) -- [Sidecar proxies](#sidecar-proxy-requirements) -- [Exported services](#exported-service-requirements) -- [ACLs](#acl-requirements) +## Mesh gateway specifications -### Mesh gateway requirements +To change Consul's default configuration and enable cluster peering through mesh gateways, use a mesh configuration entry to update your network's service mesh proxies globally: -Mesh gateways are required for routing service mesh traffic between partitions with cluster peering connections. Consider the following general requirements for mesh gateways when using cluster peering: +1. In a `mesh` configuration entry, set `PeerThroughMeshGateways` to `true`: -- A cluster requires a registered mesh gateway in order to export services to peers. -- For Enterprise, this mesh gateway must also be registered in the same partition as the exported services and their `exported-services` configuration entry. -- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. + + + ```hcl + Kind = "mesh" + Peering { + PeerThroughMeshGateways = true + } + ``` + + -In addition, you must define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. +1. Write the configuration entry to Consul: -#### Mesh gateway modes + ```shell + $ consul config write mesh-config.hcl + ``` -By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. +When cluster peering through mesh gateways, consider the following deployment requirements: + +- A cluster requires a registered mesh gateway in order to export services to peers in other regions or cloud providers. +- The mesh gateway must also be registered in the same admin partition as the exported services and their `exported-services` configuration entry. An enterprise license is required to use multiple admin partitions with a single cluster of Consul servers. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. +- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. + +### Mesh gateway modes + +By default, cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. - For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. - The `none` mode is invalid for mesh gateways with cluster peering connections. Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) for more information. -## Sidecar proxy requirements +## Sidecar proxy specifications The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/services/usage/defin-services). @@ -52,15 +69,13 @@ The Envoy proxies that function as sidecars in your service mesh require configu - The `proxy.upstreams.destination_peer` parameter must be configured to enable cross-cluster traffic. - The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. -## Exported service requirements - -The `exported-services` configuration entry is required in order for services to communicate across partitions with cluster peering connections. +## Exported service specifications -Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/create-cluster-peering). +The `exported-services` configuration entry is required in order for services to communicate across partitions with cluster peering connections. Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering#export-services-between-clusters). Refer to the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) reference for more information. -## ACL requirements +## ACL specifications If ACLs are enabled, you must add tokens to grant the following permissions: diff --git a/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx index 29259ccf2e57..b8549c5c3fbe 100644 --- a/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx +++ b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx @@ -16,7 +16,7 @@ This page details the process for establishing a cluster peering connection betw Cluster peering between services cannot be established until all four steps are complete. -For Kubernetes-specific guidance for establishing cluster peering connections, refer to [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). +For Kubernetes guidance, refer to [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). For HCP Consul guidance, refer to [Establish cluster peering connections on HCP Consul](/hcp/docs/consul/usage/cluster-peering/create-connections). ## Requirements @@ -29,11 +29,9 @@ If you need to make services available to an admin partition in the same datacen ### Mesh gateway requirements -Mesh gateways are required for all cluster peering connections. Consider the following architectural requirements when creating mesh gateways: +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. -- A registered mesh gateway is required in order to export services to peers. -- For Enterprise, the mesh gateway that exports services must be registered in the same partition as the exported services and their `exported-services` configuration entry. -- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. +To enable cluster peering through mesh gateways and configure mesh gateways to support cluster peering, refer to [mesh gateway specifications](/consul/docs/connect/cluster-peering/tech-specs#mesh-gateway-specifications). ## Create a peering token diff --git a/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx b/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx index b984e9abde08..cfe4ba7aebc5 100644 --- a/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx +++ b/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx @@ -2,24 +2,24 @@ layout: docs page_title: Cluster Peering on Kubernetes Technical Specifications description: >- - In Kubernetes deployments, cluster peering connections interact with mesh gateways, sidecar proxies, exported services, and ACLs. Learn about requirements specific to k8s, including required Helm values and CRDs. + In Kubernetes deployments, cluster peering connections interact with mesh gateways, exported services, and ACLs. Learn about requirements specific to k8s, including required Helm values and custom resource definitions (CRDs). --- # Cluster peering on Kubernetes technical specifications -This reference topic describes the technical specifications associated with using cluster peering in your Kubernetes deployments. These specifications include [required Helm values](#helm-requirements) and [required custom resource definitions (CRDs)](#crd-requirements), as well as required Consul components and their configurations. +This reference topic describes the technical specifications associated with using cluster peering in your Kubernetes deployments. These specifications include [required Helm values](#helm-requirements) and [required custom resource definitions (CRDs)](#crd-requirements), as well as required Consul components and their configurations. To learn more about Consul's cluster peering feature, refer to [cluster peering overview](/consul/docs/connect/cluster-peering). For cluster peering requirements in non-Kubernetes deployments, refer to [cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs). -## General Requirements +## General requirements -To use cluster peering features, make sure your Consul environment meets the following prerequisites: +Make sure your Consul environment meets the following prerequisites: - Consul v1.14 or higher - Consul on Kubernetes v1.0.0 or higher - At least two Kubernetes clusters -In addition, the following service mesh components are required in order to establish cluster peering connections: +You must also configure the following service mesh components in order to establish cluster peering connections: - [Helm](#helm-requirements) - [Custom resource definitions (CRD)](#crd-requirements) @@ -27,9 +27,9 @@ In addition, the following service mesh components are required in order to esta - [Exported services](#exported-service-requirements) - [ACLs](#acl-requirements) -## Helm requirements +## Helm specifications - Mesh gateways are required when establishing cluster peering connections. The following values must be set in the Helm chart to enable mesh gateways: +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. The following values must be set in the Helm chart to enable mesh gateways: - [`global.tls.enabled = true`](/consul/docs/k8s/helm#v-global-tls-enabled) - [`meshGateway.enabled = true`](/consul/docs/k8s/helm#v-meshgateway-enabled) @@ -54,7 +54,7 @@ meshGateway: After mesh gateways are enabled in the Helm chart, you can separately [configure Mesh CRDs](#mesh-gateway-configuration-for-kubernetes). -## CRD requirements +## CRD specifications You must create the following CRDs in order to establish a peering connection: @@ -100,28 +100,9 @@ spec: -## Mesh gateway requirements +## Mesh gateway specifications -Mesh gateways are required for routing service mesh traffic between partitions with cluster peering connections. Consider the following general requirements for mesh gateways when using cluster peering: - -- A cluster requires a registered mesh gateway in order to export services to peers. -- For Enterprise, this mesh gateway must also be registered in the same partition as the exported services and their `exported-services` configuration entry. -- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. - -In addition, you must define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. - -### Mesh gateway modes - -By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. - -- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. -- The `none` mode is invalid for mesh gateways with cluster peering connections. - -Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) for more information. - -### Mesh gateway configuration for Kubernetes - -Mesh gateways are required for cluster peering connections. Complete the following steps to add mesh gateways to your deployment so that you can establish cluster peering connections: +To change Consul's default configuration and enable cluster peering through mesh gateways, use a mesh configuration entry to update your network's service mesh proxies globally: 1. In `cluster-01` create the `Mesh` custom resource with `peeringThroughMeshGateways` set to `true`. @@ -139,39 +120,47 @@ Mesh gateways are required for cluster peering connections. Complete the followi -1. Apply the mesh gateway to `cluster-01`. Replace `CLUSTER1_CONTEXT` to target the first Consul cluster. +1. Apply the mesh CRD to `cluster-01`. ```shell-session $ kubectl --context $CLUSTER1_CONTEXT apply -f mesh.yaml ``` -1. Repeat the process to create and apply a mesh gateway with cluster peering enabled to `cluster-02`. Replace `CLUSTER2_CONTEXT` to target the second Consul cluster. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: Mesh - metadata: - name: mesh - spec: - peering: - peerThroughMeshGateways: true - ``` - - +1. Apply the mesh CRD to `cluster-02`. ```shell-session $ kubectl --context $CLUSTER2_CONTEXT apply -f mesh.yaml ``` -## Exported service requirements + + + For help setting up the cluster context variables used in this example, refer to [assign cluster IDs to environmental variables](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#assign-cluster-ids-to-environmental-variables). + + + +When cluster peering through mesh gateways, consider the following deployment requirements: + +- A Consul cluster requires a registered mesh gateway in order to export services to peers in other regions or cloud providers. +- The mesh gateway must also be registered in the same admin partition as the exported services and their `exported-services` configuration entry. An enterprise license is required to use multiple admin partitions with a single cluster of Consul servers. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. +- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For additional Envoy proxy configuration information, refer to [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides). + +### Mesh gateway modes + +By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. + +- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. +- The `none` mode is invalid for mesh gateways with cluster peering connections. + +To learn how to change the mesh gateway mode to `local` on your Kubernetes deployment, refer to [configure the mesh gateway mode for traffic between services](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#configure-the-mesh-gateway-mode-for-traffic-between-services). + +## Exported service specifications -The `exported-services` CRD is required in order for services to communicate across partitions with cluster peering connections. +The `exported-services` CRD is required in order for services to communicate across partitions with cluster peering connections. Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#export-services-between-clusters). -Refer to the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) reference for more information. +Refer to [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) for more information. -## ACL requirements +## ACL specifications If ACLs are enabled, you must add tokens to grant the following permissions: diff --git a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx index d4c8a3fa9eae..436998ad9c74 100644 --- a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx +++ b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx @@ -5,17 +5,20 @@ description: -> Learn how to dynamically query the Consul DNS using prepared queries, which enable robust service and node lookups. --- -# Enable dynamic DNS aueries +# Enable dynamic DNS queries This topic describes how to dynamically query the Consul catalog using prepared queries. Prepared queries are configurations that enable you to register a complex service query and execute it on demand. For information about how to perform standard node and service lookups, refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups). ## Introduction + Prepared queries provide a rich set of lookup features, such as filtering by multiple tags and automatically failing over to look for services in remote datacenters if no healthy nodes are available in the local datacenter. You can also create prepared query templates that match names using a prefix match, allowing a single template to apply to potentially many services. Refer to [Query Consul Nodes and Services Overview](/consul/docs/services/discovery/dns-overview) for additional information about DNS query behaviors. ## Requirements + Consul 0.6.4 or later is required to create prepared query templates. ### ACLs + If ACLs are enabled, the querying service must present a token linked to permissions that enable read access for query, service, and node resources. Refer to the following documentation for information about creating policies to enable read access to the necessary resources: - [Prepared query rules](/consul/docs/security/acl/acl-rules#prepared-query-rules) @@ -23,6 +26,7 @@ If ACLs are enabled, the querying service must present a token linked to permiss - [Node rules](/consul/docs/security/acl/acl-rules#node-rules) ## Create prepared queries + Refer to the [prepared query reference](/consul/api-docs/query#create-prepared-query) for usage information. 1. Specify the prepared query options in JSON format. The following prepared query targets all instances of the `redis` service in `dc1` and `dc2`: @@ -64,13 +68,15 @@ Refer to the [prepared query reference](/consul/api-docs/query#create-prepared-q $ curl --request POST --data @payload.json http://127.0.0.1:8500/v1/query {"ID":"014af5ff-29e6-e972-dcf8-6ee602137127"}% ``` -1. To run the query, send a GET request to the endpoint and specify the ID returned from the POST call. + +1. To run the query, send a GET request to the endpoint and specify the ID returned from the POST call. ```shell-session $ curl http://127.0.0.1:8500/v1/query/14af5ff-29e6-e972-dcf8-6ee602137127/execute\?near\=_agent ``` ## Execute prepared queries + You can execute a prepared query using the standard lookup format or the strict RFC 2782 SRV lookup. ### Standard lookup @@ -84,6 +90,7 @@ Use the following format to execute a prepared query using the standard lookup f Refer [Standard lookups](/consul/docs/services/discovery/dns-static-lookups#standard-lookups) for additional information about the standard lookup format in Consul. ### RFC 2782 SRV lookup + Use the following format to execute a prepared query using the RFC 2782 lookup format: ``` @@ -93,9 +100,11 @@ _._tcp.query[.]. For additional information about following the RFC 2782 SRV lookup format in Consul, refer to [RFC 2782 Lookup](/consul/docs/services/discovery/dns-static-lookups#rfc-2782-lookup). For general information about the RFC 2782 specification, refer to [A DNS RR for specifying the location of services \(DNS SRV\)](https://tools.ietf.org/html/rfc2782). ### Lookup options -The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of this Consul agent. + +The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of this Consul agent. The `query name` or `id` subdomain is the name or ID of an existing prepared query. ## Results -To allow for simple load balancing, Consul returns the set of nodes in random order for each query. Prepared queries support A and SRV records. SRV records provide the port that a service is registered. Consul only serves SRV records if the client specifically requests them. \ No newline at end of file + +To allow for simple load balancing, Consul returns the set of nodes in random order for each query. Prepared queries support A and SRV records. SRV records provide the port that a service is registered. Consul only serves SRV records if the client specifically requests them. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 67317ed5fa77..ec1afef6f1f1 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -1180,10 +1180,6 @@ { "title": "Cluster Peering", "routes": [ - { - "title": "Overview", - "href": "/docs/connect/cluster-peering" - }, { "title": "Technical Specifications", "path": "k8s/connect/cluster-peering/tech-specs" From 5560dcdbaecafc78a1475084af86a7b03db992e9 Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:16:33 -0500 Subject: [PATCH 136/421] =?UTF-8?q?Revert=20"cache:=20refactor=20agent=20c?= =?UTF-8?q?ache=20fetching=20to=20prevent=20unnecessary=20f=E2=80=A6=20(#1?= =?UTF-8?q?6818)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "cache: refactor agent cache fetching to prevent unnecessary fetches on error (#14956)" --- .changelog/16818.txt | 3 + agent/agent_test.go | 12 +- agent/cache/cache.go | 304 +++++++++--------- agent/cache/cache_test.go | 33 +- agent/cache/entry.go | 6 +- agent/cache/watch.go | 4 +- .../TestRuntimeConfig_Sanitize.golden | 2 - agent/testagent.go | 3 - 8 files changed, 176 insertions(+), 191 deletions(-) create mode 100644 .changelog/16818.txt diff --git a/.changelog/16818.txt b/.changelog/16818.txt new file mode 100644 index 000000000000..665c11034cc6 --- /dev/null +++ b/.changelog/16818.txt @@ -0,0 +1,3 @@ +```release-note:bug +cache: revert cache refactor which could cause blocking queries to never return +``` diff --git a/agent/agent_test.go b/agent/agent_test.go index bcc57cf41e0c..55b161b52481 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -4819,19 +4819,19 @@ services { deadlineCh := time.After(10 * time.Second) start := time.Now() +LOOP: for { select { case evt := <-ch: // We may receive several notifications of an error until we get the // first successful reply. require.Equal(t, "foo", evt.CorrelationID) - if evt.Err == nil { - require.NoError(t, evt.Err) - require.NotNil(t, evt.Result) - t.Logf("took %s to get first success", time.Since(start)) - return + if evt.Err != nil { + break LOOP } - t.Logf("saw error: %v", evt.Err) + require.NoError(t, evt.Err) + require.NotNil(t, evt.Result) + t.Logf("took %s to get first success", time.Since(start)) case <-deadlineCh: t.Fatal("did not get notified successfully") } diff --git a/agent/cache/cache.go b/agent/cache/cache.go index 55b1654af26e..ea537cc9e6f7 100644 --- a/agent/cache/cache.go +++ b/agent/cache/cache.go @@ -84,8 +84,8 @@ var Counters = []prometheus.CounterDefinition{ // Constants related to refresh backoff. We probably don't ever need to // make these configurable knobs since they primarily exist to lower load. const ( - DefaultCacheRefreshBackoffMin = 3 // 3 attempts before backing off - DefaultCacheRefreshMaxWait = 1 * time.Minute // maximum backoff wait time + CacheRefreshBackoffMin = 3 // 3 attempts before backing off + CacheRefreshMaxWait = 1 * time.Minute // maximum backoff wait time // The following constants are default values for the cache entry // rate limiter settings. @@ -138,7 +138,10 @@ type Cache struct { entriesLock sync.RWMutex entries map[string]cacheEntry entriesExpiryHeap *ttlcache.ExpiryHeap - lastGoroutineID uint64 + + fetchLock sync.Mutex + lastFetchID uint64 + fetchHandles map[string]fetchHandle // stopped is used as an atomic flag to signal that the Cache has been // discarded so background fetches and expiry processing should stop. @@ -151,6 +154,11 @@ type Cache struct { rateLimitCancel context.CancelFunc } +type fetchHandle struct { + id uint64 + stopCh chan struct{} +} + // typeEntry is a single type that is registered with a Cache. type typeEntry struct { // Name that was used to register the Type @@ -196,13 +204,6 @@ type Options struct { EntryFetchMaxBurst int // EntryFetchRate represents the max calls/sec for a single cache entry EntryFetchRate rate.Limit - - // CacheRefreshBackoffMin is the number of attempts to wait before backing off. - // Mostly configurable just for testing. - CacheRefreshBackoffMin uint - // CacheRefreshMaxWait is the maximum backoff wait time. - // Mostly configurable just for testing. - CacheRefreshMaxWait time.Duration } // Equal return true if both options are equivalent @@ -218,12 +219,6 @@ func applyDefaultValuesOnOptions(options Options) Options { if options.EntryFetchMaxBurst == 0 { options.EntryFetchMaxBurst = DefaultEntryFetchMaxBurst } - if options.CacheRefreshBackoffMin == 0 { - options.CacheRefreshBackoffMin = DefaultCacheRefreshBackoffMin - } - if options.CacheRefreshMaxWait == 0 { - options.CacheRefreshMaxWait = DefaultCacheRefreshMaxWait - } if options.Logger == nil { options.Logger = hclog.New(nil) } @@ -239,6 +234,7 @@ func New(options Options) *Cache { types: make(map[string]typeEntry), entries: make(map[string]cacheEntry), entriesExpiryHeap: ttlcache.NewExpiryHeap(), + fetchHandles: make(map[string]fetchHandle), stopCh: make(chan struct{}), options: options, rateLimitContext: ctx, @@ -408,23 +404,11 @@ func (c *Cache) getEntryLocked( // Check if re-validate is requested. If so the first time round the // loop is not a hit but subsequent ones should be treated normally. if !tEntry.Opts.Refresh && info.MustRevalidate { - // It is important to note that this block ONLY applies when we are not - // in indefinite refresh mode (where the underlying goroutine will - // continue to re-query for data). - // - // In this mode goroutines have a 1:1 relationship to RPCs that get - // executed, and importantly they DO NOT SLEEP after executing. - // - // This means that a running goroutine for this cache entry extremely - // strongly implies that the RPC has not yet completed, which is why - // this check works for the revalidation-avoidance optimization here. - if entry.GoroutineID != 0 { - // There is an active goroutine performing a blocking query for - // this data, which has not returned. - // - // We can logically deduce that the contents of the cache are - // actually current, and we can simply return this while leaving - // the blocking query alone. + if entry.Fetching { + // There is an active blocking query for this data, which has not + // returned. We can logically deduce that the contents of the cache + // are actually current, and we can simply return this while + // leaving the blocking query alone. return true, true, entry } return true, false, entry @@ -554,7 +538,7 @@ RETRY_GET: // At this point, we know we either don't have a value at all or the // value we have is too old. We need to wait for new data. - waiterCh := c.fetch(key, r) + waiterCh := c.fetch(key, r, true, 0, false) // No longer our first time through first = false @@ -581,36 +565,46 @@ func makeEntryKey(t, dc, peerName, token, key string) string { return fmt.Sprintf("%s/%s/%s/%s", t, dc, token, key) } -// fetch triggers a new background fetch for the given Request. If a background -// fetch is already running or a goroutine to manage that still exists for a -// matching Request, the waiter channel for that request is returned. The -// effect of this is that there is only ever one blocking query and goroutine -// for any matching requests. -func (c *Cache) fetch(key string, r getOptions) <-chan struct{} { +// fetch triggers a new background fetch for the given Request. If a +// background fetch is already running for a matching Request, the waiter +// channel for that request is returned. The effect of this is that there +// is only ever one blocking query for any matching requests. +// +// If allowNew is true then the fetch should create the cache entry +// if it doesn't exist. If this is false, then fetch will do nothing +// if the entry doesn't exist. This latter case is to support refreshing. +func (c *Cache) fetch(key string, r getOptions, allowNew bool, attempt uint, ignoreExisting bool) <-chan struct{} { + // We acquire a write lock because we may have to set Fetching to true. c.entriesLock.Lock() defer c.entriesLock.Unlock() - ok, entryValid, entry := c.getEntryLocked(r.TypeEntry, key, r.Info) - switch { - case ok && entryValid: - // This handles the case where a fetch succeeded after checking for its - // existence in getWithIndex. This ensures that we don't miss updates. + // This handles the case where a fetch succeeded after checking for its existence in + // getWithIndex. This ensures that we don't miss updates. + if ok && entryValid && !ignoreExisting { ch := make(chan struct{}) close(ch) return ch + } - case ok && entry.GoroutineID != 0: - // If we already have an entry and there's a goroutine to keep it - // refreshed then don't spawn another one to do the same work. - // - // Return the currently active waiter. + // If we aren't allowing new values and we don't have an existing value, + // return immediately. We return an immediately-closed channel so nothing + // blocks. + if !ok && !allowNew { + ch := make(chan struct{}) + close(ch) + return ch + } + + // If we already have an entry and it is actively fetching, then return + // the currently active waiter. + if ok && entry.Fetching { return entry.Waiter + } - case !ok: - // If we don't have an entry, then create it. The entry must be marked - // as invalid so that it isn't returned as a valid value for a zero - // index. + // If we don't have an entry, then create it. The entry must be marked + // as invalid so that it isn't returned as a valid value for a zero index. + if !ok { entry = cacheEntry{ Valid: false, Waiter: make(chan struct{}), @@ -621,100 +615,27 @@ func (c *Cache) fetch(key string, r getOptions) <-chan struct{} { } } - // Assign each background fetching goroutine a unique ID and fingerprint - // the cache entry with the same ID. This way if the cache entry is ever - // cleaned up due to expiry and later recreated the old goroutine can - // detect that and terminate rather than leak and do double work. - c.lastGoroutineID++ - entry.GoroutineID = c.lastGoroutineID + // Set that we're fetching to true, which makes it so that future + // identical calls to fetch will return the same waiter rather than + // perform multiple fetches. + entry.Fetching = true c.entries[key] = entry metrics.SetGauge([]string{"consul", "cache", "entries_count"}, float32(len(c.entries))) metrics.SetGauge([]string{"cache", "entries_count"}, float32(len(c.entries))) - // The actual Fetch must be performed in a goroutine. - go c.launchBackgroundFetcher(entry.GoroutineID, key, r) - - return entry.Waiter -} - -func (c *Cache) launchBackgroundFetcher(goroutineID uint64, key string, r getOptions) { - defer func() { - c.entriesLock.Lock() - defer c.entriesLock.Unlock() - entry, ok := c.entries[key] - if ok && entry.GoroutineID == goroutineID { - entry.GoroutineID = 0 - c.entries[key] = entry - } - }() - - var attempt uint - for { - shouldStop, shouldBackoff := c.runBackgroundFetcherOnce(goroutineID, key, r) - if shouldStop { - return - } - - if shouldBackoff { - attempt++ - } else { - attempt = 0 - } - // If we're over the attempt minimum, start an exponential backoff. - wait := backOffWait(c.options, attempt) - - // If we have a timer, wait for it - wait += r.TypeEntry.Opts.RefreshTimer - - select { - case <-time.After(wait): - case <-c.stopCh: - return // Check if cache was stopped - } - - // Trigger. - r.Info.MustRevalidate = false - r.Info.MinIndex = 0 - - // We acquire a write lock because we may have to set Fetching to true. - c.entriesLock.Lock() - - entry, ok := c.entries[key] - if !ok || entry.GoroutineID != goroutineID { - // If we don't have an existing entry, return immediately. - // - // Also if we already have an entry and it is actively fetching, then - // return immediately. - // - // If we've somehow lost control of the entry, also return. - c.entriesLock.Unlock() - return - } - - c.entries[key] = entry - metrics.SetGauge([]string{"consul", "cache", "entries_count"}, float32(len(c.entries))) - metrics.SetGauge([]string{"cache", "entries_count"}, float32(len(c.entries))) - c.entriesLock.Unlock() - } -} - -func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOptions) (shouldStop, shouldBackoff bool) { - // Freshly re-read this, rather than relying upon the caller to fetch it - // and pass it in. - c.entriesLock.RLock() - entry, ok := c.entries[key] - c.entriesLock.RUnlock() + tEntry := r.TypeEntry - if !ok || entry.GoroutineID != goroutineID { - // If we don't have an existing entry, return immediately. - // - // Also if something weird has happened to orphan this goroutine, also - // return immediately. - return true, false - } + // The actual Fetch must be performed in a goroutine. Ensure that we only + // have one in-flight at a time, but don't use a deferred + // context.WithCancel style termination so that these things outlive their + // requester. + // + // By the time we get here the system WANTS to make a replacement fetcher, so + // we terminate the prior one and replace it. + handle := c.getOrReplaceFetchHandle(key) + go func(handle fetchHandle) { + defer c.deleteFetchHandle(key, handle.id) - tEntry := r.TypeEntry - { // NOTE: this indentation is here to facilitate the PR review diff only // If we have background refresh and currently are in "disconnected" state, // waiting for a response might mean we mark our results as stale for up to // 10 minutes (max blocking timeout) after connection is restored. To reduce @@ -728,7 +649,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp c.entriesLock.Lock() defer c.entriesLock.Unlock() entry, ok := c.entries[key] - if !ok || entry.RefreshLostContact.IsZero() || entry.GoroutineID != goroutineID { + if !ok || entry.RefreshLostContact.IsZero() { return } entry.RefreshLostContact = time.Time{} @@ -752,15 +673,12 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp Index: entry.Index, } } - if err := entry.FetchRateLimiter.Wait(c.rateLimitContext); err != nil { if connectedTimer != nil { connectedTimer.Stop() } entry.Error = fmt.Errorf("rateLimitContext canceled: %s", err.Error()) - // NOTE: this can only happen when the entire cache is being - // shutdown and isn't something that can happen normally. - return true, false + return } // Start building the new entry by blocking on the fetch. result, err := r.Fetch(fOpts) @@ -768,8 +686,17 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp connectedTimer.Stop() } + // If we were stopped while waiting on a blocking query now would be a + // good time to detect that. + select { + case <-handle.stopCh: + return + default: + } + // Copy the existing entry to start. newEntry := entry + newEntry.Fetching = false // Importantly, always reset the Error. Having both Error and a Value that // are non-nil is allowed in the cache entry but it indicates that the Error @@ -825,7 +752,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp if result.Index > 0 { // Reset the attempts counter so we don't have any backoff - shouldBackoff = false + attempt = 0 } else { // Result having a zero index is an implicit error case. There was no // actual error but it implies the RPC found in index (nothing written @@ -840,7 +767,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // state it can be considered a bug in the RPC implementation (to ever // return a zero index) however since it can happen this is a safety net // for the future. - shouldBackoff = true + attempt++ } // If we have refresh active, this successful response means cache is now @@ -860,7 +787,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp metrics.IncrCounterWithLabels([]string{"cache", tEntry.Name, "fetch_error"}, 1, labels) // Increment attempt counter - shouldBackoff = true + attempt++ // If we are refreshing and just failed, updated the lost contact time as // our cache will be stale until we get successfully reconnected. We only @@ -877,7 +804,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // Set our entry c.entriesLock.Lock() - if currEntry, ok := c.entries[key]; !ok || currEntry.GoroutineID != goroutineID { + if _, ok := c.entries[key]; !ok { // This entry was evicted during our fetch. DON'T re-insert it or fall // through to the refresh loop below otherwise it will live forever! In // theory there should not be any Get calls waiting on entry.Waiter since @@ -890,7 +817,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // Trigger any waiters that are around. close(entry.Waiter) - return true, false + return } // If this is a new entry (not in the heap yet), then setup the @@ -915,22 +842,79 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // request back up again shortly but in the general case this prevents // spamming the logs with tons of ACL not found errors for days. if tEntry.Opts.Refresh && !preventRefresh { - return false, shouldBackoff + // Check if cache was stopped + if atomic.LoadUint32(&c.stopped) == 1 { + return + } + + // If we're over the attempt minimum, start an exponential backoff. + wait := backOffWait(attempt) + + // If we have a timer, wait for it + wait += tEntry.Opts.RefreshTimer + + select { + case <-time.After(wait): + case <-handle.stopCh: + return + } + + // Trigger. The "allowNew" field is false because in the time we were + // waiting to refresh we may have expired and got evicted. If that + // happened, we don't want to create a new entry. + r.Info.MustRevalidate = false + r.Info.MinIndex = 0 + c.fetch(key, r, false, attempt, true) } + }(handle) + + return entry.Waiter +} + +func (c *Cache) getOrReplaceFetchHandle(key string) fetchHandle { + c.fetchLock.Lock() + defer c.fetchLock.Unlock() + + if prevHandle, ok := c.fetchHandles[key]; ok { + close(prevHandle.stopCh) } - return true, false + c.lastFetchID++ + + handle := fetchHandle{ + id: c.lastFetchID, + stopCh: make(chan struct{}), + } + + c.fetchHandles[key] = handle + + return handle +} + +func (c *Cache) deleteFetchHandle(key string, fetchID uint64) { + c.fetchLock.Lock() + defer c.fetchLock.Unlock() + + // Only remove a fetchHandle if it's YOUR fetchHandle. + handle, ok := c.fetchHandles[key] + if !ok { + return + } + + if handle.id == fetchID { + delete(c.fetchHandles, key) + } } -func backOffWait(opts Options, failures uint) time.Duration { - if failures > opts.CacheRefreshBackoffMin { - shift := failures - opts.CacheRefreshBackoffMin - waitTime := opts.CacheRefreshMaxWait +func backOffWait(failures uint) time.Duration { + if failures > CacheRefreshBackoffMin { + shift := failures - CacheRefreshBackoffMin + waitTime := CacheRefreshMaxWait if shift < 31 { waitTime = (1 << shift) * time.Second } - if waitTime > opts.CacheRefreshMaxWait { - waitTime = opts.CacheRefreshMaxWait + if waitTime > CacheRefreshMaxWait { + waitTime = CacheRefreshMaxWait } return waitTime + lib.RandomStagger(waitTime) } diff --git a/agent/cache/cache_test.go b/agent/cache/cache_test.go index 98b04ee9a488..6f8805be06d8 100644 --- a/agent/cache/cache_test.go +++ b/agent/cache/cache_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/lib/ttlcache" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/sdk/testutil/retry" ) // Test a basic Get with no indexes (and therefore no blocking queries). @@ -1750,12 +1751,22 @@ func TestCache_RefreshLifeCycle(t *testing.T) { require.NoError(t, err) require.Equal(t, true, result) + waitUntilFetching := func(expectValue bool) { + retry.Run(t, func(t *retry.R) { + c.entriesLock.Lock() + defer c.entriesLock.Unlock() + entry, ok := c.entries[key] + require.True(t, ok) + if expectValue { + require.True(t, entry.Fetching) + } else { + require.False(t, entry.Fetching) + } + }) + } + // ensure that the entry is fetching again - c.entriesLock.Lock() - entry, ok := c.entries[key] - require.True(t, ok) - require.True(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(true) requestChan := make(chan error) @@ -1789,11 +1800,7 @@ func TestCache_RefreshLifeCycle(t *testing.T) { } // ensure that the entry is fetching again - c.entriesLock.Lock() - entry, ok = c.entries[key] - require.True(t, ok) - require.True(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(true) // background a call that will wait for a newer version - will result in an acl not found error go getError(5) @@ -1814,11 +1821,7 @@ func TestCache_RefreshLifeCycle(t *testing.T) { // ensure that the ACL not found error killed off the background refresh // but didn't remove it from the cache - c.entriesLock.Lock() - entry, ok = c.entries[key] - require.True(t, ok) - require.False(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(false) } type fakeType struct { diff --git a/agent/cache/entry.go b/agent/cache/entry.go index 7130381dea45..0c71e9443713 100644 --- a/agent/cache/entry.go +++ b/agent/cache/entry.go @@ -26,9 +26,9 @@ type cacheEntry struct { Index uint64 // Metadata that is used for internal accounting - Valid bool // True if the Value is set - GoroutineID uint64 // Nonzero if a fetch goroutine is running. - Waiter chan struct{} // Closed when this entry is invalidated + Valid bool // True if the Value is set + Fetching bool // True if a fetch is already active + Waiter chan struct{} // Closed when this entry is invalidated // Expiry contains information about the expiration of this // entry. This is a pointer as its shared as a value in the diff --git a/agent/cache/watch.go b/agent/cache/watch.go index d87bca38a544..abd247c4f1c5 100644 --- a/agent/cache/watch.go +++ b/agent/cache/watch.go @@ -137,7 +137,7 @@ func (c *Cache) notifyBlockingQuery(ctx context.Context, r getOptions, correlati failures = 0 } else { failures++ - wait = backOffWait(c.options, failures) + wait = backOffWait(failures) c.options.Logger. With("error", err). @@ -224,7 +224,7 @@ func (c *Cache) notifyPollingQuery(ctx context.Context, r getOptions, correlatio // as this would eliminate the single-flighting of these requests in the cache and // the efficiencies gained by it. if failures > 0 { - wait = backOffWait(c.options, failures) + wait = backOffWait(failures) } else { // Calculate when the cached data's Age will get too stale and // need to be re-queried. When the data's Age already exceeds the diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index 2c5b91c98a53..fff09fa343f6 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -79,8 +79,6 @@ "BootstrapExpect": 0, "BuildDate": "2019-11-20 05:00:00 +0000 UTC", "Cache": { - "CacheRefreshBackoffMin": 0, - "CacheRefreshMaxWait": "0s", "EntryFetchMaxBurst": 42, "EntryFetchRate": 0.334, "Logger": null diff --git a/agent/testagent.go b/agent/testagent.go index 76d82a2f84d2..f81b56314f9c 100644 --- a/agent/testagent.go +++ b/agent/testagent.go @@ -211,9 +211,6 @@ func (a *TestAgent) Start(t *testing.T) error { } else { result.RuntimeConfig.Telemetry.Disable = true } - // Lower the maximum backoff period of a cache refresh just for - // tests see #14956 for more. - result.RuntimeConfig.Cache.CacheRefreshMaxWait = 1 * time.Second // Lower the resync interval for tests. result.RuntimeConfig.LocalProxyConfigResyncInterval = 250 * time.Millisecond From 064fb1c15c976990e09e143059d1efa8325e4f1f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 30 Mar 2023 13:24:06 -0400 Subject: [PATCH 137/421] Backport of ci: changes resulting from running on consul-enterprise into release/1.15.x (#16822) * backport of commit 77bafd8821f11ab1fea0dad79000290cde4f5c9e * backport of commit ccf59a4f947e30f61563d8f42e8114f52603c996 --------- Co-authored-by: John Murret --- .github/workflows/build-distros.yml | 23 ++++- .github/workflows/go-tests.yml | 100 ++++++++++++++++---- .github/workflows/reusable-check-go-mod.yml | 12 ++- .github/workflows/reusable-dev-build.yml | 12 ++- .github/workflows/reusable-lint.yml | 15 ++- .github/workflows/reusable-unit-split.yml | 32 +++++-- .github/workflows/reusable-unit.yml | 44 +++++---- 7 files changed, 185 insertions(+), 53 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 620504e4e4d1..8b6c6d9930f9 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -28,6 +28,9 @@ jobs: uses: ./.github/workflows/reusable-check-go-mod.yml with: runs-on: ${{ needs.setup.outputs.compute-medium }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} build-386: needs: @@ -38,6 +41,12 @@ jobs: runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -56,6 +65,12 @@ jobs: runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -75,11 +90,17 @@ jobs: GOOS: linux steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' - run: | - sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list # workaround for https://github.com/actions/runner-images/issues/1983 sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 9c799fa9a4cb..f2dcd6bad15a 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -39,13 +39,20 @@ jobs: uses: ./.github/workflows/reusable-check-go-mod.yml with: runs-on: ${{ needs.setup.outputs.compute-small }} - + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + check-generated-protobuf: needs: - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -62,7 +69,7 @@ jobs: - run: make proto-lint name: "Protobuf Lint" - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh check-generated-deep-copy: needs: @@ -70,6 +77,10 @@ jobs: runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -82,34 +93,42 @@ jobs: exit 1 fi - name: Notify Slack - if: failure() - run: .github/scripts/notify_slack.sh + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + lint-enums: needs: - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh lint-container-test-deps: - needs: - - setup + needs: + - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' - - run: make lint-container-test-deps + - run: make lint-container-test-deps - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh lint-consul-retry: @@ -118,12 +137,16 @@ jobs: runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh lint: @@ -132,6 +155,9 @@ jobs: uses: ./.github/workflows/reusable-lint.yml with: runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} lint-32bit: needs: @@ -140,7 +166,9 @@ jobs: with: go-arch: "386" runs-on: ${{ needs.setup.outputs.compute-xl }} - + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} # create a development build dev-build: @@ -149,7 +177,10 @@ jobs: uses: ./.github/workflows/reusable-dev-build.yml with: runs-on: ${{ needs.setup.outputs.compute-xl }} - + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # TODO(JM): - linux arm64 is not available in our self-hosted runners # they are currently on the roadmap. # # create a development build for arm64 @@ -183,7 +214,10 @@ jobs: directory: . runner-count: 12 runs-on: ${{ needs.setup.outputs.compute-xl }} - + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-race: needs: @@ -195,6 +229,10 @@ jobs: go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"' package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul/command/'" runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-32bit: needs: @@ -206,24 +244,36 @@ jobs: go-arch: "386" go-test-flags: 'export GO_TEST_FLAGS="-short"' runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-envoyextensions: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: directory: envoyextensions runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-troubleshoot: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: directory: troubleshoot runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-api-1-19: needs: @@ -233,6 +283,10 @@ jobs: with: directory: api runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-api-1-20: needs: @@ -242,6 +296,10 @@ jobs: with: directory: api runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-sdk-1-19: needs: @@ -251,6 +309,10 @@ jobs: with: directory: sdk runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} go-test-sdk-1-20: needs: @@ -260,6 +322,10 @@ jobs: with: directory: sdk runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} noop: runs-on: ubuntu-latest diff --git a/.github/workflows/reusable-check-go-mod.yml b/.github/workflows/reusable-check-go-mod.yml index ab6ef2420d6a..2078b0c3217d 100644 --- a/.github/workflows/reusable-check-go-mod.yml +++ b/.github/workflows/reusable-check-go-mod.yml @@ -7,12 +7,22 @@ on: description: An expression indicating which kind of runners to use. required: true type: string + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true jobs: check-go-mod: runs-on: ${{ fromJSON(inputs.runs-on) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -24,5 +34,5 @@ jobs: exit 1 fi - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml index d4832fb0a98f..1a0483b19350 100644 --- a/.github/workflows/reusable-dev-build.yml +++ b/.github/workflows/reusable-dev-build.yml @@ -11,11 +11,21 @@ on: description: An expression indicating which kind of runners to use. required: true type: string + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true jobs: build: runs-on: ${{ fromJSON(inputs.runs-on) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -27,5 +37,5 @@ jobs: name: ${{inputs.uploaded-binary-name}} path: ./bin/consul - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-lint.yml b/.github/workflows/reusable-lint.yml index 121d9231fa63..82650fd5e903 100644 --- a/.github/workflows/reusable-lint.yml +++ b/.github/workflows/reusable-lint.yml @@ -11,9 +11,14 @@ on: description: An expression indicating which kind of runners to use. required: true type: string - + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true env: - GOTAGS: "" # No tags for OSS but there are for enterprise + GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" GOARCH: ${{inputs.go-arch}} jobs: @@ -32,6 +37,10 @@ jobs: name: lint ${{ matrix.directory }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' @@ -43,5 +52,5 @@ jobs: version: v1.51 args: --build-tags="${{ env.GOTAGS }}" -v - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml index fbd4d859bf52..5049cfaa5f37 100644 --- a/.github/workflows/reusable-unit-split.yml +++ b/.github/workflows/reusable-unit-split.yml @@ -30,12 +30,21 @@ on: required: false type: string default: "" - + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true + consul-license: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 GOARCH: ${{inputs.go-arch}} TOTAL_RUNNERS: ${{inputs.runner-count}} + CONSUL_LICENSE: ${{secrets.consul-license}} + GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" jobs: set-test-package-matrix: @@ -67,15 +76,14 @@ jobs: echo "Hard limits" ulimit -Ha - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' cache: true - - name: Install gotestsum - run: | - wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz - sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz - rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz - run: mkdir -p ${{env.TEST_RESULTS}} - name: go mod download working-directory: ${{inputs.directory}} @@ -84,9 +92,13 @@ jobs: uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{inputs.uploaded-binary-name}} - path: /usr/local/bin + path: ${{inputs.directory}} + - name: Display downloaded file + run: ls -ld consul + working-directory: ${{inputs.directory}} + - run: echo "$GITHUB_WORKSPACE/${{inputs.directory}}" >> $GITHUB_PATH - name: Make sure consul is executable - run: sudo chmod +x /usr/local/bin/consul + run: chmod +x $GITHUB_WORKSPACE/${{inputs.directory}}/consul - run: go env - name: Run tests working-directory: ${{inputs.directory}} @@ -100,7 +112,7 @@ jobs: # some tests expect this umask, and arm images have a different default umask 0022 - gotestsum \ + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --format=short-verbose \ --jsonfile /tmp/jsonfile/go-test.log \ --debug \ @@ -124,5 +136,5 @@ jobs: run: | .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml index 24c20abe52d4..6f3bc242e7db 100644 --- a/.github/workflows/reusable-unit.yml +++ b/.github/workflows/reusable-unit.yml @@ -26,34 +26,37 @@ on: required: false type: string default: "" - + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true + consul-license: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 GOARCH: ${{inputs.go-arch}} + CONSUL_LICENSE: ${{secrets.consul-license}} + GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" jobs: go-test: runs-on: ${{ fromJSON(inputs.runs-on) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' - - name: Setup go mod cache - uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # pin@v3.2.6 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Install gotestsum - run: | - wget https://github.com/gotestyourself/gotestsum/releases/download/v${{env.GOTESTSUM_VERSION}}/gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz - sudo tar -C /usr/local/bin -xzf gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz - rm gotestsum_${{env.GOTESTSUM_VERSION}}_linux_amd64.tar.gz + go-version-file: 'go.mod' + cache: true - run: mkdir -p ${{env.TEST_RESULTS}} - name: go mod download working-directory: ${{inputs.directory}} @@ -62,12 +65,13 @@ jobs: uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 with: name: ${{inputs.uploaded-binary-name}} - path: /usr/local/bin - - name: Make sure consul is executable - run: sudo chmod +x /usr/local/bin/consul + path: ${{inputs.directory}} - name: Display downloaded file run: ls -ld consul - working-directory: /usr/local/bin + working-directory: ${{inputs.directory}} + - run: echo "$GITHUB_WORKSPACE/${{inputs.directory}}" >> $GITHUB_PATH + - name: Make sure consul is executable + run: chmod +x $GITHUB_WORKSPACE/${{inputs.directory}}/consul - run: go env - name: Run tests working-directory: ${{inputs.directory}} @@ -79,7 +83,7 @@ jobs: ${{inputs.go-test-flags}} - gotestsum \ + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --format=short-verbose \ --jsonfile /tmp/jsonfile/go-test.log \ --debug \ @@ -103,5 +107,5 @@ jobs: run: | .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails - name: Notify Slack - if: failure() + if: ${{ failure() }} run: .github/scripts/notify_slack.sh From 22cdccb7ff0e9cc2ea7ad5f99e6f285f0cdf21e4 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 30 Mar 2023 17:23:56 -0400 Subject: [PATCH 138/421] Backport of docs: raise awareness of GH-16779 into release/1.15.x (#16828) * backport of commit cb2d7880ef11fa34e40c869075e2b5e882cec9ce * backport of commit 4ef063b94995c37cc0c019a004765f31cae0daa8 --------- Co-authored-by: Jared Kirschner --- CHANGELOG.md | 4 ++++ .../docs/release-notes/consul/v1_15_x.mdx | 11 ++++++++++- .../content/docs/upgrading/upgrade-specific.mdx | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 485d26865235..6b85536e5c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ BUG FIXES: ## 1.15.0 (February 23, 2023) +KNOWN ISSUES: + +* connect: An issue with leaf certificate rotation can cause some service instances to lose their ability to communicate in the mesh after 72 hours (LeafCertTTL). This issue is not consistently reproducible. We are working to address this issue in an upcoming patch release. To err on the side of caution, service mesh deployments should not upgrade to Consul v1.15 at this time. Refer to [[GH-16779](https://github.com/hashicorp/consul/issues/16779)] for the latest information. + BREAKING CHANGES: * acl errors: Delete and get requests now return descriptive errors when the specified resource cannot be found. Other ACL request errors provide more information about when a resource is missing. Add error for when the ACL system has not been bootstrapped. diff --git a/website/content/docs/release-notes/consul/v1_15_x.mdx b/website/content/docs/release-notes/consul/v1_15_x.mdx index 5d8ffd9dc21c..99c2961bd4ab 100644 --- a/website/content/docs/release-notes/consul/v1_15_x.mdx +++ b/website/content/docs/release-notes/consul/v1_15_x.mdx @@ -66,7 +66,16 @@ For more detailed information, please refer to the [upgrade details page](/consu ## Known Issues -The following issues are known to exist in the v1.15.0 release: +The following issues are known to exist in the v1.15.x releases: + +- All current 1.15.x versions are under investigation for a not-consistently-reproducible + issue that can cause some service instances to lose their ability to communicate in the mesh after + [72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) + due to a problem with leaf certificate rotation. + We will update this section with more information as our investigation continues, + including the target availability for a fix. + Refer to [GH-16779](https://github.com/hashicorp/consul/issues/16779) + for the latest information. - For v1.15.0, Consul is reporting newer releases of Envoy (for example, v1.25.1) as not supported, even though these versions are listed as valid in the [Envoy compatilibity matrix](/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent). The following error would result for newer versions of Envoy: diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 06997760e35c..936a4cec491b 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -16,6 +16,22 @@ upgrade flow. ## Consul 1.15.x +#### Service mesh known issue + +To err on the side of caution, +service mesh deployments should not upgrade to Consul v1.15 at this time. + +We are currently investigating a not-consistently-reproducible issue that can cause +some service instances to lose their ability to communicate in the mesh after +[72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) +due to a problem with leaf certificate rotation. +We will update this section with more information as our investigation continues, +including the target availability for a fix. + +If you are already operating Consul v1.15, refer to discussion of this issue on +[GH-16779](https://github.com/hashicorp/consul/issues/16779) +for potential workarounds and to share your observations. + #### Removing configuration options The `connect.enable_serverless_plugin` configuration option was removed. Lambda integration is now enabled by default. From ca148fc0c058b8961a4f267118c4c8cc927a028f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 30 Mar 2023 23:59:21 -0400 Subject: [PATCH 139/421] backport of commit a362862cf6004a2bb4806579d28eb0ce2503d11e (#16832) Co-authored-by: John Murret --- .github/workflows/go-tests.yml | 2 +- .github/workflows/reusable-unit.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index f2dcd6bad15a..67f10a4721cf 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -227,7 +227,7 @@ jobs: with: directory: . go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"' - package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul/command/'" + package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'" runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} secrets: diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml index 6f3bc242e7db..b474b4c69bd3 100644 --- a/.github/workflows/reusable-unit.yml +++ b/.github/workflows/reusable-unit.yml @@ -50,9 +50,6 @@ jobs: - name: Setup Git if: ${{ endsWith(inputs.repository-name, '-enterprise') }} run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" - - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 - with: - go-version-file: 'go.mod' - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' From 6cbd5035e5c028350b61e5ad1cf4e35e571792a3 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 31 Mar 2023 10:55:25 -0400 Subject: [PATCH 140/421] Backport of Fix broken links in Consul docs into release/1.15.x (#16768) * backport of commit fba9e901d6b086ab5544d5d442d064324bf5f404 * backport of commit fbdeaf2ebe6a77e26e79bbb0003eaa2b05ca1a12 * cherry pick and fix merge conflict --------- Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> Co-authored-by: Tu Nguyen --- website/content/docs/agent/config/cli-flags.mdx | 4 ++-- website/content/docs/agent/config/config-files.mdx | 2 +- website/content/docs/agent/index.mdx | 3 ++- website/content/docs/agent/wal-logstore/enable.mdx | 2 +- website/content/docs/agent/wal-logstore/monitoring.mdx | 2 +- website/content/docs/api-gateway/configuration/routes.mdx | 4 ++-- .../docs/api-gateway/usage/route-to-peered-services.mdx | 2 +- website/content/docs/architecture/scale.mdx | 2 +- website/content/docs/connect/cluster-peering/tech-specs.mdx | 2 +- .../connect/cluster-peering/usage/manage-connections.mdx | 2 +- .../content/docs/connect/config-entries/service-defaults.mdx | 4 ++-- website/content/docs/connect/dataplane/consul-dataplane.mdx | 2 +- .../gateways/api-gateway/configuration/http-route.mdx | 2 +- website/content/docs/connect/proxies/envoy.mdx | 4 ++-- website/content/docs/connect/proxies/integrate.mdx | 2 +- website/content/docs/ecs/terraform/secure-configuration.mdx | 5 +++-- website/content/docs/enterprise/license/faq.mdx | 4 ++-- website/content/docs/install/cloud-auto-join.mdx | 2 +- website/content/docs/integrate/download-tools.mdx | 3 +-- website/content/docs/integrate/partnerships.mdx | 2 +- website/content/docs/internals/acl.mdx | 5 +---- .../docs/k8s/deployment-configurations/consul-enterprise.mdx | 2 +- website/content/docs/k8s/dns.mdx | 2 +- website/content/docs/k8s/index.mdx | 2 +- website/content/docs/k8s/installation/install.mdx | 2 +- .../content/docs/release-notes/consul-api-gateway/v0_4_x.mdx | 2 +- website/content/docs/release-notes/consul/v1_12_x.mdx | 2 +- .../content/docs/security/acl/auth-methods/kubernetes.mdx | 5 ++--- website/content/docs/security/acl/auth-methods/oidc.mdx | 2 +- .../content/docs/services/discovery/dns-static-lookups.mdx | 2 +- website/content/docs/services/usage/checks.mdx | 2 +- website/content/docs/services/usage/define-services.mdx | 2 +- .../content/docs/services/usage/register-services-checks.mdx | 4 ++-- .../content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx | 4 ++-- 34 files changed, 45 insertions(+), 48 deletions(-) diff --git a/website/content/docs/agent/config/cli-flags.mdx b/website/content/docs/agent/config/cli-flags.mdx index 3cdfd6fe91b5..bff79cc63520 100644 --- a/website/content/docs/agent/config/cli-flags.mdx +++ b/website/content/docs/agent/config/cli-flags.mdx @@ -157,7 +157,7 @@ information. - `-segment` ((#\_segment)) - This flag is used to set the name of the network segment the agent belongs to. An agent can only join and communicate with other agents within its network segment. Ensure the [join - operation uses the correct port for this segment](/consul/docs/enterprise/network-segments#join_a_client_to_a_segment). + operation uses the correct port for this segment](/consul/docs/enterprise/network-segments/create-network-segment#configure-clients-to-join-segments). Review the [Network Segments documentation](/consul/docs/enterprise/network-segments/create-network-segment) for more details. By default, this is an empty string, which is the `` network segment. @@ -490,7 +490,7 @@ information. the data directory. This is useful when running multiple Consul agents on the same host for testing. This defaults to false in Consul prior to version 0.8.5 and in 0.8.5 and later defaults to true, so you must opt-in for host-based IDs. Host-based - IDs are generated using [gopsutil](https://github.com/shirou/gopsutil/tree/master/v3/host), which + IDs are generated using [gopsutil](https://github.com/shirou/gopsutil/), which is shared with HashiCorp's [Nomad](https://www.nomadproject.io/), so if you opt-in to host-based IDs then Consul and Nomad will use information on the host to automatically assign the same ID in both systems. diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 705d0e837802..4ba7ba54313e 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -924,7 +924,7 @@ Refer to the [formatting specification](https://golang.org/pkg/time/#ParseDurati [`acl.tokens.agent_recovery`](#acl_tokens_agent_recovery).** - `config_file_service_registration` ((#acl_tokens_config_file_service_registration)) - Specifies the ACL - token the agent uses to register services and checks from [service](/consul/docs/services/usage/define-services) and [check](/consul/docs/usage/checks) definitions + token the agent uses to register services and checks from [service](/consul/docs/services/usage/define-services) and [check](/consul/docs/services/usage/checks) definitions specified in configuration files or fragments passed to the agent using the `-hcl` flag. diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index d1860b10a508..51c5b713ef5e 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -62,7 +62,8 @@ its consequences during outage situations). Reaping is similar to leaving, causing all associated services to be deregistered. ## Limit traffic rates -You can define a set of rate limiting configurations that help operators protect Consul servers from excessive or peak usage. The configurations enable you to gracefully degrade Consul servers to avoid a global interruption of service. You can allocate a set of resources to different Consul users and eliminate the risks that some users consuming too many resources pose to others. Consul supports global server rate limiting, which lets configure Consul servers to deny requests that exceed the read or write limits. Refer to [Traffic Rate Limits Overview](/consul/docs/agent/limits/limit-traffic-rates). + +You can define a set of rate limiting configurations that help operators protect Consul servers from excessive or peak usage. The configurations enable you to gracefully degrade Consul servers to avoid a global interruption of service. Consul supports global server rate limiting, which lets configure Consul servers to deny requests that exceed the read or write limits. Refer to [Traffic Rate Limits Overview](/consul/docs/agent/limits). ## Requirements diff --git a/website/content/docs/agent/wal-logstore/enable.mdx b/website/content/docs/agent/wal-logstore/enable.mdx index cdd9b933dfd5..2a339fcf89e1 100644 --- a/website/content/docs/agent/wal-logstore/enable.mdx +++ b/website/content/docs/agent/wal-logstore/enable.mdx @@ -23,7 +23,7 @@ The overall process for enabling the WAL LogStore backend for one server consist ## Requirements -- Consul v1.15 or later is required for all servers in the datacenter. Refer to the [standard upgrade procedure](/consul/docs/upgrading/general-process) and the [1.15 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-15-x) for additional information. +- Consul v1.15 or later is required for all servers in the datacenter. Refer to the [standard upgrade procedure](/consul/docs/upgrading/instructions/general-process) and the [1.15 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-15-x) for additional information. - A Consul cluster with at least three nodes are required to safely test the WAL backend without downtime. We recommend taking the following additional measures: diff --git a/website/content/docs/agent/wal-logstore/monitoring.mdx b/website/content/docs/agent/wal-logstore/monitoring.mdx index 5be765cf408b..f4f81a986d27 100644 --- a/website/content/docs/agent/wal-logstore/monitoring.mdx +++ b/website/content/docs/agent/wal-logstore/monitoring.mdx @@ -7,7 +7,7 @@ description: >- # Monitor Raft metrics and logs for WAL -This topic describes how to monitor Raft metrics and logs if you are testing the WAL backend. We strongly recommend monitoring the Consul cluster, especially the target server, for evidence that the WAL backend is not functioning correctly. Refer to [Enable the experimental WAL LogStore backend](/consul/docs/agent/wal-logstore/index) for additional information about the WAL backend. +This topic describes how to monitor Raft metrics and logs if you are testing the WAL backend. We strongly recommend monitoring the Consul cluster, especially the target server, for evidence that the WAL backend is not functioning correctly. Refer to [Enable the experimental WAL LogStore backend](/consul/docs/agent/wal-logstore/enable) for additional information about the WAL backend. !> **Upgrade warning:** The WAL LogStore backend is experimental. diff --git a/website/content/docs/api-gateway/configuration/routes.mdx b/website/content/docs/api-gateway/configuration/routes.mdx index 4702a3608311..27b520340248 100644 --- a/website/content/docs/api-gateway/configuration/routes.mdx +++ b/website/content/docs/api-gateway/configuration/routes.mdx @@ -168,7 +168,7 @@ The following example creates a route named `example-route` in namespace `gatewa ### rules.filters -The `filters` block defines steps for processing requests. You can configure filters to modify the properties of matching incoming requests and enable Consul API Gateway features, such as rewriting path prefixes (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information). +The `filters` block defines steps for processing requests. You can configure filters to modify the properties of matching incoming requests and enable Consul API Gateway features, such as rewriting path prefixes (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information). * Type: Array of objects * Required: Optional @@ -203,7 +203,7 @@ Specifies rules for rewriting the URL of incoming requests when `rules.filters.t ### rules.filters.urlRewrite.path -Specifies a list of objects that determine how Consul API Gateway rewrites URL paths (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information). +Specifies a list of objects that determine how Consul API Gateway rewrites URL paths (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information). The following table describes the parameters for `path`: diff --git a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx index fe8ca69732cc..509247623974 100644 --- a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx +++ b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx @@ -18,7 +18,7 @@ This topic describes how to configure Consul API Gateway to route traffic to ser ## Configuration -Specify the following fields in your `MeshService` configuration to use this feature. Refer to the [MeshService configuration reference](/consul/docs/api-gateway/configuration/mesh) for details about the parameters. +Specify the following fields in your `MeshService` configuration to use this feature. Refer to the [MeshService configuration reference](/consul/docs/api-gateway/configuration/meshservice) for details about the parameters. - [`name`](/consul/docs/api-gateway/configuration/meshservice#name) - [`peer`](/consul/docs/api-gateway/configuration/meshservice#peer) diff --git a/website/content/docs/architecture/scale.mdx b/website/content/docs/architecture/scale.mdx index f05c4b094cfc..da2031fd95ba 100644 --- a/website/content/docs/architecture/scale.mdx +++ b/website/content/docs/architecture/scale.mdx @@ -51,7 +51,7 @@ To mitigate these risks, we recommend a maximum of 5,000 Consul client agents in 1. Run exactly one Consul agent per host in the infrastructure. 1. Break up the single Consul datacenter into multiple smaller datacenters. -1. Enterprise users can define [network segments](/consul/docs/enterprise/network-segments) to divide the single gossip pool in the Consul datacenter into multiple smaller pools. +1. Enterprise users can define [network segments](/consul/docs/enterprise/network-segments/network-segments-overview) to divide the single gossip pool in the Consul datacenter into multiple smaller pools. If appropriate for your use case, we recommend breaking up a single Consul datacenter into multiple smaller datacenters. Running multiple datacenters reduces your network’s blast radius more than applying network segments. diff --git a/website/content/docs/connect/cluster-peering/tech-specs.mdx b/website/content/docs/connect/cluster-peering/tech-specs.mdx index 2dd335407e6c..929a25f924dc 100644 --- a/website/content/docs/connect/cluster-peering/tech-specs.mdx +++ b/website/content/docs/connect/cluster-peering/tech-specs.mdx @@ -62,7 +62,7 @@ Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) ## Sidecar proxy specifications -The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/services/usage/defin-services). +The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/services/usage/define-services). - Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams`](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) documentation for details. - The `proxy.upstreams.destination_name` parameter is always required. diff --git a/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx index d2e3b77181fc..a4e92373328a 100644 --- a/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx +++ b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx @@ -125,7 +125,7 @@ For more information, including optional flags and parameters, refer to the [`co $ curl --request DELETE --header "X-Consul-Token: b23b3cad-5ea1-4413-919e-c76884b9ad60" http://127.0.0.1:8500/v1/peering/cluster-02 ``` -This endpoint does not return a response. For more information, including optional parameters, refer to the [`/peering` endpoint reference](/consul/api-docs/peering/consul/api-docs/peering#delete-a-peering-connection). +This endpoint does not return a response. For more information, including optional parameters, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#delete-a-peering-connection). diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 75b8497c98c2..0f9f7cfa9789 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -697,7 +697,7 @@ You can configure the following parameters in the `EnvoyExtensions` block: ### `Destination[]` -Configures the destination for service traffic through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. +Configures the destination for service traffic through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/gateways/terminating-gateway) for additional information. You can configure the following parameters in the `Destination` block: @@ -1082,7 +1082,7 @@ You can configure the following parameters in the `EnvoyExtensions` block: ### `spec.destination` -Map of configurations that specify one or more destinations for service traffic routed through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/terminating-gateway) for additional information. +Map of configurations that specify one or more destinations for service traffic routed through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/gateways/terminating-gateway) for additional information. #### Values diff --git a/website/content/docs/connect/dataplane/consul-dataplane.mdx b/website/content/docs/connect/dataplane/consul-dataplane.mdx index b4661d861234..ab59a5ba60cc 100644 --- a/website/content/docs/connect/dataplane/consul-dataplane.mdx +++ b/website/content/docs/connect/dataplane/consul-dataplane.mdx @@ -7,7 +7,7 @@ description: >- # Consul Dataplane CLI Reference -The `consul-dataplane` command interacts with the binary for [simplified service mesh with Consul Dataplane](/consul/docs/k8s/dataplane). Use this command to install Consul Dataplane, configure its Envoy proxies, and secure Dataplane deployments. +The `consul-dataplane` command interacts with the binary for [simplified service mesh with Consul Dataplane](/consul/docs/connect/dataplane). Use this command to install Consul Dataplane, configure its Envoy proxies, and secure Dataplane deployments. ## Usage diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx index 7ab1a506d462..997e2bbf692e 100644 --- a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx +++ b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx @@ -435,7 +435,7 @@ Specifies rule for rewriting the URL of incoming requests when an incoming reque ### Rules[].Filters.URLRewrite.Path -Specifies a path that determines how Consul API Gateway rewrites a URL path. Refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information. +Specifies a path that determines how Consul API Gateway rewrites a URL path. Refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information. #### Values diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index bb27db3b2a31..73e7e3c535c0 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -111,7 +111,7 @@ If TLS is enabled on Consul, you will also need to add the following environment - [`CONSUL_CACERT`](/consul/commands#consul_cacert) - [`CONSUL_CLIENT_CERT`](/consul/commands#consul_client_cert) -- [`CONSUL_CLIENT_KEY`](//consulcommands#consul_client_key) +- [`CONSUL_CLIENT_KEY`](/consul/commands#consul_client_key) - [`CONSUL_HTTP_SSL`](/consul/commands#consul_http_ssl) ## Bootstrap Configuration @@ -194,7 +194,7 @@ The [Advanced Configuration](#advanced-configuration) section describes addition ### Bootstrap Envoy on Windows VMs -> Complete the [Connect Services on Windows Workloads to Consul Service Mesh tutorial](https://consul.io/consu/tutorials/consul-windows-workloads?utm_source=docs) to learn how to deploy Consul and use its service mesh on Windows VMs. +> Complete the [Connect Services on Windows Workloads to Consul Service Mesh tutorial](/consul/tutorials/developer-mesh/consul-windows-workloads) to learn how to deploy Consul and use its service mesh on Windows VMs. If you are running Consul on a Windows VM, attempting to bootstrap Envoy with the `consul connect envoy` command returns the following output: diff --git a/website/content/docs/connect/proxies/integrate.mdx b/website/content/docs/connect/proxies/integrate.mdx index 20a203059d05..24b9208af61a 100644 --- a/website/content/docs/connect/proxies/integrate.mdx +++ b/website/content/docs/connect/proxies/integrate.mdx @@ -138,7 +138,7 @@ documentation for details about supported configuration parameters. ### Service Discovery -Proxies can use Consul's [service discovery API](https://consul.io/%60/v1/health/connect/:service_id%60) to return all available, Connect-capable endpoints for a given service. This endpoint supports a `cached` query parameter, which uses [agent caching](/consul/api-docs/features/caching) to improve +Proxies can use Consul's [service discovery API](/consul/api-docs/health#list-service-instances-for-connect-enabled-service) to return all available, Connect-capable endpoints for a given service. This endpoint supports a `cached` query parameter, which uses [agent caching](/consul/api-docs/features/caching) to improve performance. The API package provides a [`UseCache`] query option to leverage caching. In addition to performance improvements, using the cache makes the mesh more resilient to Consul server outages. This is because the mesh "fails static" with the last known set of service instances still used, rather than errors on new connections. diff --git a/website/content/docs/ecs/terraform/secure-configuration.mdx b/website/content/docs/ecs/terraform/secure-configuration.mdx index c1997241c3b1..07031f5490e3 100644 --- a/website/content/docs/ecs/terraform/secure-configuration.mdx +++ b/website/content/docs/ecs/terraform/secure-configuration.mdx @@ -108,8 +108,9 @@ The following table describes the required input variables for the `acl-controll | `name_prefix` | string | AWS resources created by the `acl-controller` module will include this prefix in the resource name. | -If you are using Consul Enterprise, see Admin Partitions and Namespaces for -additional configuration required to support Consul Enterprise on ECS. + +If you are using Consul Enterprise, see the [Admin Partitions and Namespaces requirements documentation](/consul/docs/ecs/requirements) for additional configuration required to support Consul Enterprise on ECS. + ## Deploy your services diff --git a/website/content/docs/enterprise/license/faq.mdx b/website/content/docs/enterprise/license/faq.mdx index 4221762b2dfa..d8a46ee20934 100644 --- a/website/content/docs/enterprise/license/faq.mdx +++ b/website/content/docs/enterprise/license/faq.mdx @@ -138,7 +138,7 @@ When a customer deploys new clusters to a 1.10.0+ent release, they need to have New Consul cluster deployments using 1.10.0+ent will need to have a valid license on servers to successfully deploy. This valid license must be on-disk (auto-loaded) or as an environment variable. -Please see the [upgrade requirements](https://consul.io/faq#q-what-are-the-upgrade-requirements). +Please see the [upgrade requirements](/consul/docs/enterprise/license/faq#q-what-are-the-upgrade-requirements). ## Q: What is the migration path for customers who want to migrate from their existing license-as-applied-via-the-CLI flow to the license on disk flow? @@ -183,7 +183,7 @@ When downgrading to a version of Consul before 1.10.0+ent, customers will need t ## Q: Are there potential pitfalls when downgrading or upgrading Consul server instances? -~> Verify that you meet the [upgrade requirements](https://consul.io/faq#q-what-are-the-upgrade-requirements). +~> Verify that you meet the [upgrade requirements](/consul/docs/enterprise/license/faq#q-what-are-the-upgrade-requirements). Assume a scenario where there are three Consul server nodes: diff --git a/website/content/docs/install/cloud-auto-join.mdx b/website/content/docs/install/cloud-auto-join.mdx index 40e2f44edf02..82cf6b6d5929 100644 --- a/website/content/docs/install/cloud-auto-join.mdx +++ b/website/content/docs/install/cloud-auto-join.mdx @@ -34,7 +34,7 @@ or via a configuration file: ## Auto-join with Network Segments -In order to use cloud auto-join with [Network Segments](/consul/docs/enterprise/network-segments), +In order to use cloud auto-join with [Network Segments](/consul/docs/enterprise/network-segments/network-segments-overview), you must reconfigure the Consul agent's Serf LAN port to match that of the segment you wish to join. diff --git a/website/content/docs/integrate/download-tools.mdx b/website/content/docs/integrate/download-tools.mdx index a1263b7d817e..b348e82a66da 100644 --- a/website/content/docs/integrate/download-tools.mdx +++ b/website/content/docs/integrate/download-tools.mdx @@ -15,7 +15,7 @@ These Consul tools are created and managed by the dedicated engineers at HashiCo - [Envconsul](https://github.com/hashicorp/envconsul) - Read and set environmental variables for processes from Consul. - [Consul API Gateway](https://github.com/hashicorp/consul-api-gateway/) - dedicated ingress solution for intelligently routing traffic to applications running on a Consul Service Mesh. -- [Consul ESM](https://github.com/hashicorp/consul-esm) - Provides external service monitoring for Consul. Complete the [tutorial](https://consul.io/(https://learn.hashicorp.com/tutorials/consul/service-registration-external-services?utm_source=docs)) to learn more. +- [Consul ESM](https://github.com/hashicorp/consul-esm) - Provides external service monitoring for Consul. Complete the [tutorial](/consul/tutorials/developer-discovery/service-registration-external-services?utm_source=docs) to learn more. - [Consul Migrate](https://github.com/hashicorp/consul-migrate) - Data migration tool to handle Consul upgrades to 0.5.1+ - [Consul Replicate](https://github.com/hashicorp/consul-replicate) - Consul cross-DC KV replication daemon. - [Consul Template](https://github.com/hashicorp/consul-template) - Generic template rendering and notifications with Consul. Complete the [tutorial](/consul/tutorials/developer-configuration/consul-template?utm_source=docs) to the learn more. @@ -54,7 +54,6 @@ These Consul tools are created and managed by the amazing members of the Consul - [gradle-consul-plugin](https://github.com/amirkibbar/red-apple) - A Consul Gradle plugin - [hashi-ui](https://github.com/jippi/hashi-ui) - A modern user interface for the Consul and Nomad - [HashiBox](https://github.com/nunchistudio/hashibox) - Vagrant environment to simulate highly-available cloud with Consul, Nomad, Vault, and optional support for Waypoint. OSS & Enterprise supported. -- [helios-consul](https://github.com/SVT/helios-consul) - Service registrar plugin for Helios - [Jenkins Consul Plugin](https://plugins.jenkins.io/consul) - Jenkins plugin for service discovery and K/V store - [marathon-consul](https://github.com/allegro/marathon-consul) - Service registry bridge for Marathon - [marathon-consul](https://github.com/CiscoCloud/marathon-consul) - Bridge from Marathon apps to the Consul K/V store diff --git a/website/content/docs/integrate/partnerships.mdx b/website/content/docs/integrate/partnerships.mdx index 0f9f7301b784..34bcfe2631fe 100644 --- a/website/content/docs/integrate/partnerships.mdx +++ b/website/content/docs/integrate/partnerships.mdx @@ -89,7 +89,7 @@ Here are links to resources, documentation, examples and best practices to guide - [Monitoring Consul with Datadog APM](https://www.datadoghq.com/blog/consul-datadog/) - [Monitor HCP Consul with New Relic Instant Observability](https://github.com/newrelic-experimental/hashicorp-quickstart-annex/blob/main/hcp-consul/README.md) - [HCP Consul and CloudFabrix AIOps Integration](https://bot-docs.cloudfabrix.io/Bots/consul/?h=consul) -- [Consul and SnappyFlow Full Stack Observability](https://docs.snappyflow.io/docs/integrations/hcp_consul) +- [Consul and SnappyFlow Full Stack Observability](https://docs.snappyflow.io/docs/Integrations/hcp_consul) **Network Performance Monitoring (NPM)** diff --git a/website/content/docs/internals/acl.mdx b/website/content/docs/internals/acl.mdx index 05c8d3801914..87e35844179a 100644 --- a/website/content/docs/internals/acl.mdx +++ b/website/content/docs/internals/acl.mdx @@ -10,7 +10,4 @@ description: >- # ACL System ((#version_8_acls)) -This content has been moved into the [ACL Guide](/consul/tutorials/security/access-control-setup-production). - -See [Complete ACL Coverage in Consul 0.8](/consul/docs/security/acl/acl-legacy) for details -about ACL changes in Consul 0.8 and later. +This content has been moved into the [ACL Guide](/consul/tutorials/security/access-control-setup-production). \ No newline at end of file diff --git a/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx b/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx index 48c4db1fa0cd..29314c943541 100644 --- a/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx +++ b/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx @@ -11,7 +11,7 @@ You can use this Helm chart to deploy Consul Enterprise by following a few extra Find the license file that you received in your welcome email. It should have a `.hclic` extension. You will use the contents of this file to create a Kubernetes secret before installing the Helm chart. --> **Note:** This guide assumes you are storing your license as a Kubernetes Secret. If you would like to store the enterprise license in Vault, please reference [Storing the Enterprise License in Vault](/consul/docs/k8s/deployment-configuration/vault/data-integration/enterprise-license). +-> **Note:** This guide assumes you are storing your license as a Kubernetes Secret. If you would like to store the enterprise license in Vault, please reference [Storing the Enterprise License in Vault](/consul/docs/k8s/deployment-configurations/vault/data-integration/enterprise-license). You can use the following commands to create the secret with name `consul-ent-license` and key `key`: diff --git a/website/content/docs/k8s/dns.mdx b/website/content/docs/k8s/dns.mdx index 4262523a83d8..0f34dd250783 100644 --- a/website/content/docs/k8s/dns.mdx +++ b/website/content/docs/k8s/dns.mdx @@ -11,7 +11,7 @@ One of the primary query interfaces to Consul is the [DNS interface](/consul/docs/services/discovery/dns-overview). You can configure Consul DNS in Kubernetes using a [stub-domain configuration](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configure-stub-domain-and-upstream-dns-servers) -if using KubeDNS or a [proxy configuration](https://coredns.io/plugins/proxy/) if using CoreDNS. +if using KubeDNS or a [proxy configuration](https://coredns.io/plugins/forward/) if using CoreDNS. Once configured, DNS requests in the form `.service.consul` will resolve for services in Consul. This will work from all Kubernetes namespaces. diff --git a/website/content/docs/k8s/index.mdx b/website/content/docs/k8s/index.mdx index f4d0ffe4c321..e1a8291abe3e 100644 --- a/website/content/docs/k8s/index.mdx +++ b/website/content/docs/k8s/index.mdx @@ -59,7 +59,7 @@ There are several ways to try Consul with Kubernetes in different environments. - The [Consul and Kubernetes Deployment](/consul/tutorials/kubernetes/kubernetes-deployment-guide?utm_source=docs) tutorial covers the necessary steps to install and configure a new Consul cluster on Kubernetes in production. -- The [Secure Consul and Registered Services on Kubernetes](https://consul.io/consul/tutorials/kubernetes/kubernetes-secure-agents?utm_source=docs) tutorial covers +- The [Secure Consul and Registered Services on Kubernetes](/consul/tutorials/kubernetes/kubernetes-secure-agents?utm_source=docs) tutorial covers the necessary steps to secure a Consul cluster running on Kubernetes in production. - The [Layer 7 Observability with Consul Service Mesh](/consul/tutorials/kubernetes/kubernetes-layer7-observability) tutorial covers monitoring a diff --git a/website/content/docs/k8s/installation/install.mdx b/website/content/docs/k8s/installation/install.mdx index 2872718166c5..1d80696ba037 100644 --- a/website/content/docs/k8s/installation/install.mdx +++ b/website/content/docs/k8s/installation/install.mdx @@ -85,7 +85,7 @@ or read the [Helm Chart Reference](/consul/docs/k8s/helm). ### Minimal `values.yaml` for Consul service mesh -The following `values.yaml` config file contains the minimum required settings to enable [Consul Service Mesh](https://consul.io/(/docs/k8s/connect)): +The following `values.yaml` config file contains the minimum required settings to enable [Consul Service Mesh](/consul/docs/k8s/connect): diff --git a/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx b/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx index a6e82167d30d..71fa38c8965d 100644 --- a/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx +++ b/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx @@ -31,7 +31,7 @@ description: >- to rewrite the URL path in a client's HTTP request before sending the request to a service. For example, you could configure the gateway to change the path from `//store/checkout` to `//cart/checkout`. Refer to the [usage - documentation](/consul/docs/api-gateway/usage) for additional information. + documentation](/consul/docs/connect/gateways/api-gateway/usage) for additional information. ## What has Changed diff --git a/website/content/docs/release-notes/consul/v1_12_x.mdx b/website/content/docs/release-notes/consul/v1_12_x.mdx index c1d0d4724d3f..ebdaaed98b0f 100644 --- a/website/content/docs/release-notes/consul/v1_12_x.mdx +++ b/website/content/docs/release-notes/consul/v1_12_x.mdx @@ -13,7 +13,7 @@ description: >- - **Per listener TLS Config**: It is now possible to configure TLS differently for each of Consul's listeners, such as HTTPS, gRPC, and the internal multiplexed RPC listener, using the `tls` stanza. Refer to [TLS Configuration Reference](/consul/docs/agent/config/config-files#tls-configuration-reference) for more details. -- **AWS Lambda**: Adds the ability to invoke AWS Lambdas through terminating gateways, which allows for cross-datacenter communication, transparent proxy, and intentions with Consul Service Mesh. Refer to [AWS Lambda](/consul/docs]/lambda) and [Invoke Lambda Functions](/consul/docs/lambda/invocation) for more details. +- **AWS Lambda**: Adds the ability to invoke AWS Lambdas through terminating gateways, which allows for cross-datacenter communication, transparent proxy, and intentions with Consul Service Mesh. Refer to [AWS Lambda](/consul/docs/lambda) and [Invoke Lambda Functions](/consul/docs/lambda/invocation) for more details. - **Mesh-wide TLS min/max versions and cipher suites**: Using the [Mesh](/consul/docs/connect/config-entries/mesh#tls) Config Entry or CRD, it is now possible to set TLS min/max versions and cipher suites for both inbound and outbound mTLS connections. diff --git a/website/content/docs/security/acl/auth-methods/kubernetes.mdx b/website/content/docs/security/acl/auth-methods/kubernetes.mdx index b0a432884a03..13f76481c98c 100644 --- a/website/content/docs/security/acl/auth-methods/kubernetes.mdx +++ b/website/content/docs/security/acl/auth-methods/kubernetes.mdx @@ -76,14 +76,13 @@ The Kubernetes service account corresponding to the configured [`ServiceAccountJWT`](/consul/docs/security/acl/auth-methods/kubernetes#serviceaccountjwt) needs to have access to two Kubernetes APIs: -- [**TokenReview**](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#create-tokenreview-v1-authentication-k8s-io) +- [**TokenReview**](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-review-v1/) -> Kubernetes should be running with `--service-account-lookup`. This is defaulted to true in Kubernetes 1.7, but any versions prior should ensure the Kubernetes API server is started with this setting. -- [**ServiceAccount**](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#read-serviceaccount-v1-core) - (`get`) +- [**ServiceAccount**](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#service-account-tokens) The following is an example [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) diff --git a/website/content/docs/security/acl/auth-methods/oidc.mdx b/website/content/docs/security/acl/auth-methods/oidc.mdx index ea62c510083a..7d22f01df7d6 100644 --- a/website/content/docs/security/acl/auth-methods/oidc.mdx +++ b/website/content/docs/security/acl/auth-methods/oidc.mdx @@ -73,7 +73,7 @@ parameters are required to properly configure an auth method of type - `JWTSupportedAlgs` `(array)` - JWTSupportedAlgs is a list of supported signing algorithms. Defaults to `RS256`. ([Available - algorithms](https://github.com/hashicorp/consul/blob/main/vendor/github.com/coreos/go-oidc/jose.go#L7)) + algorithms](https://github.com/hashicorp/consul/blob/main/internal/go-sso/oidcauth/jwt.go)) - `BoundAudiences` `(array)` - List of `aud` claims that are valid for login; any match is sufficient. diff --git a/website/content/docs/services/discovery/dns-static-lookups.mdx b/website/content/docs/services/discovery/dns-static-lookups.mdx index aa2524ec4574..6f2db4108c37 100644 --- a/website/content/docs/services/discovery/dns-static-lookups.mdx +++ b/website/content/docs/services/discovery/dns-static-lookups.mdx @@ -327,7 +327,7 @@ This returns the unique virtual IP for any service mesh-capable service. Each se The peer name is an optional. The DNS uses it to query for the virtual IP of a service imported from the specified peer. -Consul adds virtual IPs to the [`tagged_addresses`](/consul/services/configuration/services-configuration-reference#tagged-addresses) field in the service definition under the `consul-virtual` tag. +Consul adds virtual IPs to the [`tagged_addresses`](/consul/docs/services/configuration/services-configuration-reference#tagged_addresses) field in the service definition under the `consul-virtual` tag. #### Service virtual IP lookups for Consul Enterprise diff --git a/website/content/docs/services/usage/checks.mdx b/website/content/docs/services/usage/checks.mdx index 8f86b2283975..afbf53dcc99b 100644 --- a/website/content/docs/services/usage/checks.mdx +++ b/website/content/docs/services/usage/checks.mdx @@ -245,7 +245,7 @@ Responses larger than 4KB are truncated. The HTTP response determines the status TCP checks establish connections to the specified IPs or hosts. If the check successfully establishes a connection, the service status is reported as `success`. If the IP or host does not accept the connection, the service status is reported as `critical`. We recommend TCP checks over [script checks](#script-checks) that use netcat or another external process to check a socket operation. ### TCP check configuration -Add a `tcp` field to the `check` block in your service definition file and specify the address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/health-checks-configuration) for information about all health check configurations. +Add a `tcp` field to the `check` block in your service definition file and specify the address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. In the following example, a TCP check named `SSH TCP on port 22` attempts to connect to `localhost:22` every 10 seconds: diff --git a/website/content/docs/services/usage/define-services.mdx b/website/content/docs/services/usage/define-services.mdx index 3a7862c92a73..1e0490b9b3ab 100644 --- a/website/content/docs/services/usage/define-services.mdx +++ b/website/content/docs/services/usage/define-services.mdx @@ -138,7 +138,7 @@ You can add a `check` or `checks` block to your service configuration to define ### Register a service -You can register your service using the [`consul services` command](/consul/commands/services) or by calling the [`/agent/services` API endpoint](/consul/api-docs/agent/services). Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details. +You can register your service using the [`consul services` command](/consul/commands/services) or by calling the [`/agent/services` API endpoint](/consul/api-docs/agent/service). Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details. ## Define service defaults If Consul service mesh is enabled in your network, you can define default values for services in your mesh by creating and applying a `service-defaults` configuration entry containing. Refer to [Service Mesh Configuration Overview](/consul/docs/connect/configuration) for additional information. diff --git a/website/content/docs/services/usage/register-services-checks.mdx b/website/content/docs/services/usage/register-services-checks.mdx index 89c96a1f2e7a..07a3f20ad9c0 100644 --- a/website/content/docs/services/usage/register-services-checks.mdx +++ b/website/content/docs/services/usage/register-services-checks.mdx @@ -7,7 +7,7 @@ description: -> # Register services and health checks -This topic describes how to register services and health checks with Consul in networks running on virtual machines (VM). Refer to [Define Services](/consul/usage/services/usage/define-services) and [Define Health Checks](/consul/usage/services/usage/checks) for information about how to define services and health checks. +This topic describes how to register services and health checks with Consul in networks running on virtual machines (VM). Refer to [Define Services](/consul/docs/services/usage/define-services) and [Define Health Checks](/consul/docs/services/usage/checks) for information about how to define services and health checks. ## Overview Register services and health checks in VM environments by providing the service definition to a Consul agent. You can use several different methods to register services and health checks. @@ -65,4 +65,4 @@ Send a `PUT` request to the `/agent/check/register` API endpoint to dynamically $ curl --request PUT --data @payload.json http://localhost:8500/v1/agent/check/register ``` -Refer to [Check - Agent HTTP API](/consul/api-docs/check/service) for additional information about the `check` endpoint. +Refer to [Check - Agent HTTP API](/consul/api-docs/agent/check) for additional information about the `check` endpoint. diff --git a/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx b/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx index c39aed602c06..d88560260eb9 100644 --- a/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx +++ b/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx @@ -18,8 +18,8 @@ as part of this upgrade. The 1.6.x series is the last series that had support fo ACL tokens, so this migration _must_ happen before upgrading past the 1.6.x release series. Here is some documentation that may prove useful for reference during this upgrade process: -- [ACL System in Legacy Mode](/consul/docs/security/acl/acl-legacy) - You can find - information about legacy configuration options and differences between modes here. +- [Upgrading Legacy ACL tokens](/consul/tutorials/security-operations/access-control-token-migration) - You can find + information about upgrading legacy ACL tokens and differences between modes here. - [Configuration](/consul/docs/agent/config) - You can find more details around legacy ACL and new ACL configuration options here. Legacy ACL config options will be listed as deprecates as of 1.4.0. From 1469efe81938052c534fa84f50e1fc7b01ec2888 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 3 Apr 2023 14:28:40 -0400 Subject: [PATCH 141/421] Backport of Allow dialer to re-establish terminated peering into release/1.15.x (#16859) * backport of commit ea941524972371f19fde314aa7fa1b5a2dd7e571 * backport of commit da7507f11935b6aac6058e4b4a18e7cba28e9607 --------- Co-authored-by: freddygv --- .changelog/16776.txt | 3 +++ agent/consul/leader_peering_test.go | 24 ++++++++++++++++++++++++ agent/consul/peering_backend.go | 7 +++++-- agent/consul/peering_backend_test.go | 23 +++++++++++++++++++++-- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 .changelog/16776.txt diff --git a/.changelog/16776.txt b/.changelog/16776.txt new file mode 100644 index 000000000000..0159aee85897 --- /dev/null +++ b/.changelog/16776.txt @@ -0,0 +1,3 @@ +```release-note:improvement +peering: allow re-establishing terminated peering from new token without deleting existing peering first. +``` \ No newline at end of file diff --git a/agent/consul/leader_peering_test.go b/agent/consul/leader_peering_test.go index 143448535aa1..757fc87eebc5 100644 --- a/agent/consul/leader_peering_test.go +++ b/agent/consul/leader_peering_test.go @@ -468,6 +468,30 @@ func TestLeader_PeeringSync_Lifecycle_ServerDeletion(t *testing.T) { require.NoError(r, err) require.Equal(r, pbpeering.PeeringState_TERMINATED, peering.State) }) + + // Re-establishing a peering terminated by the acceptor should be possible + // without needing to delete the terminated peering first. + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + t.Cleanup(cancel) + + req = pbpeering.GenerateTokenRequest{ + PeerName: "my-peer-dialer", + } + resp, err = peeringClient.GenerateToken(ctx, &req) + require.NoError(t, err) + + tokenJSON, err = base64.StdEncoding.DecodeString(resp.PeeringToken) + require.NoError(t, err) + + token = structs.PeeringToken{} + require.NoError(t, json.Unmarshal(tokenJSON, &token)) + + establishReq = pbpeering.EstablishRequest{ + PeerName: "my-peer-acceptor", + PeeringToken: resp.PeeringToken, + } + _, err = dialerClient.Establish(ctx, &establishReq) + require.NoError(t, err) } func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { diff --git a/agent/consul/peering_backend.go b/agent/consul/peering_backend.go index 7c0b7c2e5424..aa18cfd04aee 100644 --- a/agent/consul/peering_backend.go +++ b/agent/consul/peering_backend.go @@ -147,8 +147,11 @@ func (b *PeeringBackend) fetchPeerServerAddresses(ws memdb.WatchSet, peerID stri if err != nil { return nil, fmt.Errorf("failed to fetch peer %q: %w", peerID, err) } - if !peering.IsActive() { - return nil, fmt.Errorf("there is no active peering for %q", peerID) + if peering == nil { + return nil, fmt.Errorf("unknown peering %q", peerID) + } + if peering.DeletedAt != nil && !structs.IsZeroProtoTime(peering.DeletedAt) { + return nil, fmt.Errorf("peering %q was deleted", peerID) } return bufferFromAddresses(peering.GetAddressesToDial()) } diff --git a/agent/consul/peering_backend_test.go b/agent/consul/peering_backend_test.go index b7a725409bc9..485806631894 100644 --- a/agent/consul/peering_backend_test.go +++ b/agent/consul/peering_backend_test.go @@ -253,7 +253,7 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { }, peerID: acceptorPeerID, expect: expectation{ - err: fmt.Sprintf(`there is no active peering for %q`, acceptorPeerID), + err: fmt.Sprintf(`unknown peering %q`, acceptorPeerID), }, }, { @@ -384,6 +384,25 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { gatewayAddrs: []string{"5.6.7.8:8443", "6.7.8.9:8443"}, }, }, + { + name: "addresses are returned if the peering is marked as terminated", + setup: func(store *state.Store) { + require.NoError(t, store.PeeringWrite(5, &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + Name: "dialer", + ID: dialerPeerID, + PeerServerAddresses: []string{"1.2.3.4:8502", "2.3.4.5:8503"}, + State: pbpeering.PeeringState_TERMINATED, + }, + })) + }, + peerID: dialerPeerID, + expect: expectation{ + // Gateways come first, and we use their LAN addresses since this is for outbound communication. + addrs: []string{"5.6.7.8:8443", "6.7.8.9:8443", "1.2.3.4:8502", "2.3.4.5:8503"}, + gatewayAddrs: []string{"5.6.7.8:8443", "6.7.8.9:8443"}, + }, + }, { name: "addresses are not returned if the peering is deleted", setup: func(store *state.Store) { @@ -401,7 +420,7 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { }, peerID: dialerPeerID, expect: expectation{ - err: fmt.Sprintf(`there is no active peering for %q`, dialerPeerID), + err: fmt.Sprintf(`peering %q was deleted`, dialerPeerID), }, }, } From 5a13be8c00ecd22df7e7de6a698da0f28b7d1b6e Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 3 Apr 2023 18:25:52 -0400 Subject: [PATCH 142/421] backport of commit c444a58ccc957c4a1574603d6778db4055f7d9ee (#16864) Co-authored-by: dttung2905 --- website/content/docs/k8s/upgrade/index.mdx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/website/content/docs/k8s/upgrade/index.mdx b/website/content/docs/k8s/upgrade/index.mdx index ae12c8e3d9f8..246197a7eed0 100644 --- a/website/content/docs/k8s/upgrade/index.mdx +++ b/website/content/docs/k8s/upgrade/index.mdx @@ -38,9 +38,9 @@ For example, if you installed Consul with `connectInject.enabled: false` and you ``` **Before performing the upgrade, be sure you read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** -~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. +~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise, Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. ### Upgrade Helm chart version @@ -87,7 +87,7 @@ If you want to upgrade to the latest `0.40.0` version, use the following procedu ``` **Before performing the upgrade, be sure you've read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** ### Upgrade Consul version @@ -126,9 +126,9 @@ to update to the new version. Before you upgrade to a new version: ``` **Before performing the upgrade, be sure you have read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** -~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. +~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise, Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. ## Determine scope of changes @@ -175,7 +175,7 @@ that can be used. Initiate the server upgrade: 1. Change the `global.image` value to the desired Consul version. -1. Set the `server.updatePartition` value to the number of server replicas. By default there are 3 servers, so you would set this value to `3`. +1. Set the `server.updatePartition` value to the number of server replicas. By default, there are 3 servers, so you would set this value to `3`. 1. Set the `updateStrategy` for clients to `OnDelete`. @@ -228,7 +228,7 @@ If you upgrade Consul from a version that uses client agents to a version the us type: OnDelete ``` -1. Add `consul.hashicorp.com/consul-k8s-version: 1.0.0` to the annotations for each pod you upgrade. +1. Add `consul.hashicorp.com/consul-k8s-version: 1.0.0` to the annotations for each pod you upgrade. 1. Follow our [recommended procedures to upgrade servers](#upgrading-consul-servers) on Kubernetes deployments to upgrade Helm values for the new version of Consul. @@ -236,7 +236,7 @@ If you upgrade Consul from a version that uses client agents to a version the us 1. Restart all gateways in your service mesh. -1. Disable client agents in your Helm chart by deleting the `client` stanza or setting `client.enabled` to `false`. +1. Disable client agents in your Helm chart by deleting the `client` stanza or setting `client.enabled` to `false`. ## Configuring TLS on an existing cluster From 45a9a266bdda7ef8b280ae2bbb4806d20f4927a8 Mon Sep 17 00:00:00 2001 From: John Murret Date: Tue, 4 Apr 2023 11:59:24 -0600 Subject: [PATCH 143/421] backport of increasing runner size for check-generated-deep-copy and lint-enums (#16870) --- .github/workflows/go-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 67f10a4721cf..0db905610051 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -74,7 +74,7 @@ jobs: check-generated-deep-copy: needs: - setup - runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. @@ -99,7 +99,7 @@ jobs: lint-enums: needs: - setup - runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. From d585c9d8182f9441918cdd5325dfa8a19799953d Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 5 Apr 2023 12:06:43 -0600 Subject: [PATCH 144/421] backport of add arm64 testing (#16883) manual creation of a failed backport --- .github/workflows/go-tests.yml | 56 ++++++++++++++---------- .github/workflows/reusable-dev-build.yml | 6 +++ 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 0db905610051..219e98db6348 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -181,30 +181,38 @@ jobs: secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - # TODO(JM): - linux arm64 is not available in our self-hosted runners - # they are currently on the roadmap. - # # create a development build for arm64 - # dev-build-arm64: - # needs: - # - setup - # uses: ./.github/workflows/reusable-dev-build.yml - # with: - # uploaded-binary-name: 'consul-bin-arm64' - # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} - - # go-test-arm64: - # # TODO(JM): Fix to run on arm64 - # needs: - # - setup - # - dev-build-arm64 - # uses: ./.github/workflows/reusable-unit-split.yml - # with: - # directory: . - # uploaded-binary-name: 'consul-bin-arm64' - # runner-count: 12 - # # runs-on: ${{ needs.setup.outputs.compute-xl-arm64 }} - # go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' - + dev-build-arm64: + # only run on enterprise because GHA does not have arm64 runners in OSS + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + uses: ./.github/workflows/reusable-dev-build.yml + with: + uploaded-binary-name: 'consul-bin-arm64' + runs-on: ${{ needs.setup.outputs.compute-xl }} + go-arch: "arm64" + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + go-test-arm64: + # only run on enterprise because GHA does not have arm64 runners in OSS + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + - dev-build-arm64 + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + uploaded-binary-name: 'consul-bin-arm64' + runner-count: 12 + runs-on: "['self-hosted', 'ondemand', 'os=macos-arm', 'arm64']" + go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + go-test: needs: - setup diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml index 1a0483b19350..36ce160bce95 100644 --- a/.github/workflows/reusable-dev-build.yml +++ b/.github/workflows/reusable-dev-build.yml @@ -14,6 +14,10 @@ on: repository-name: required: true type: string + go-arch: + required: false + type: string + default: "" secrets: elevated-github-token: required: true @@ -30,6 +34,8 @@ jobs: with: go-version-file: 'go.mod' - name: Build + env: + GOARCH: ${{ inputs.goarch }} run: make dev # save dev build to pass to downstream jobs - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 From a54f1ab03914c12bc3986aa04e626ecb5008eca3 Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Wed, 5 Apr 2023 15:29:58 -0500 Subject: [PATCH 145/421] Update release/1.15.x to reflect release of 1.15.2 (#16878) * Bump VERSION to reflect next minor release * Add changelog entry for 1.15.2 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ version/VERSION | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b85536e5c2b..5b5259a054ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +## 1.15.2 (March 30, 2023) + +FEATURES: + +* xds: Allow for configuring connect proxies to send service mesh telemetry to an HCP metrics collection service. [[GH-16585](https://github.com/hashicorp/consul/issues/16585)] + +BUG FIXES: + +* audit-logging: (Enterprise only) Fix a bug where `/agent/monitor` and `/agent/metrics` endpoints return a `Streaming not supported` error when audit logs are enabled. This also fixes the delay receiving logs when running `consul monitor` against an agent with audit logs enabled. [[GH-16700](https://github.com/hashicorp/consul/issues/16700)] +* ca: Fixes a bug where updating Vault CA Provider config would cause TLS issues in the service mesh [[GH-16592](https://github.com/hashicorp/consul/issues/16592)] +* cache: revert cache refactor which could cause blocking queries to never return [[GH-16818](https://github.com/hashicorp/consul/issues/16818)] +* gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal for TCPServices. [[GH-16781](https://github.com/hashicorp/consul/issues/16781)] +* gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal. [[GH-16651](https://github.com/hashicorp/consul/issues/16651)] +* gateway: **(Enterprise only)** Fix bug where parent refs and service refs for a route in the same namespace as the route would fallback to the default namespace if the namespace was not specified in the configuration rather than falling back to the routes namespace. [[GH-16789](https://github.com/hashicorp/consul/issues/16789)] +* gateway: **(Enterprise only)** Fix bug where routes defined in a different namespace than a gateway would fail to register. [[GH-16677](https://github.com/hashicorp/consul/pull/16677)]. +* gateways: Adds validation to ensure the API Gateway has a listener defined when created [[GH-16649](https://github.com/hashicorp/consul/issues/16649)] +* gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS. [[GH-16661](https://github.com/hashicorp/consul/issues/16661)] +* peering: **(Consul Enterprise only)** Fix issue where connect-enabled services with peer upstreams incorrectly required `service:write` access in the `default` namespace to query data, which was too restrictive. Now having `service:write` to any namespace is sufficient to query the peering data. +* peering: **(Consul Enterprise only)** Fix issue where resolvers, routers, and splitters referencing peer targets may not work correctly for non-default partitions and namespaces. Enterprise customers leveraging peering are encouraged to upgrade both servers and agents to avoid this problem. +* peering: Fix issue resulting in prepared query failover to cluster peers never un-failing over. [[GH-16729](https://github.com/hashicorp/consul/issues/16729)] +* peering: Fixes a bug that can lead to peering service deletes impacting the state of local services [[GH-16570](https://github.com/hashicorp/consul/issues/16570)] +* peering: Fixes a bug where the importing partition was not added to peered failover targets, which causes issues when the importing partition is a non-default partition. [[GH-16675](https://github.com/hashicorp/consul/issues/16675)] +* raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. [[GH-16647](https://github.com/hashicorp/consul/issues/16647)] +* ui: fix PUT token request with adding missed AccessorID property to requestBody [[GH-16660](https://github.com/hashicorp/consul/issues/16660)] +* ui: fix rendering issues on Overview and empty-states by addressing isHTMLSafe errors [[GH-16574](https://github.com/hashicorp/consul/issues/16574)] + ## 1.15.1 (March 7, 2023) IMPROVEMENTS: diff --git a/version/VERSION b/version/VERSION index 827d8d6a3bfa..749afc16bf35 100644 --- a/version/VERSION +++ b/version/VERSION @@ -1 +1 @@ -1.15.2-dev +1.15.3-dev From df8c4f8318290e08602e9304bd424956a9499487 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 5 Apr 2023 19:05:08 -0400 Subject: [PATCH 146/421] backport of commit dd0216a6144aebc2f8b1b8271dc57a8f7fd3e76f (#16894) Co-authored-by: Dan Bond --- .github/workflows/frontend.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index e7d9e385514a..094d86ad5555 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -41,6 +41,9 @@ jobs: with: node-version: '16' + - name: Install Yarn + run: npm install -g yarn + # Install dependencies. - name: install yarn packages working-directory: ui @@ -58,6 +61,9 @@ jobs: with: node-version: '16' + - name: Install Yarn + run: npm install -g yarn + # Install dependencies. - name: install yarn packages working-directory: ui @@ -68,11 +74,11 @@ jobs: ember-build-test: needs: setup - runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} env: - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary - EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam - CONSUL_NSPACES_ENABLED: 0 # NOTE: this should be 1 in ENT. + EMBER_TEST_REPORT: test-results/report-oss.xml # outputs test report for CircleCI test summary + EMBER_TEST_PARALLEL: true # enables test parallelization with ember-exam + CONSUL_NSPACES_ENABLED: ${{ endsWith(github.repository, '-enterprise') && 1 || 0 }} # NOTE: this should be 1 in ENT. steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -80,6 +86,12 @@ jobs: with: node-version: '16' + - name: Install Yarn + run: npm install -g yarn + + - name: Install Chrome + uses: browser-actions/setup-chrome@29abc1a83d1d71557708563b4bc962d0f983a376 # pin@v1.2.1 + # Install dependencies. - name: install yarn packages working-directory: ui From cedcd1119f9f8ab900ff011b070996d6f1607132 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 5 Apr 2023 19:32:03 -0400 Subject: [PATCH 147/421] no-op commit due to failed cherry-picking (#16902) Co-authored-by: temp From a3d3607d3b4a209bc9d4c7f719cc4f23ac591d13 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 5 Apr 2023 19:33:26 -0400 Subject: [PATCH 148/421] always test oss and conditionally test enterprise (#16827) (#16897) Co-authored-by: John Murret --- .github/workflows/go-tests.yml | 28 +++++++++++++++++++++-- .github/workflows/reusable-unit-split.yml | 6 ++++- .github/workflows/reusable-unit.yml | 6 ++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 219e98db6348..ca90ac31d342 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -212,8 +212,23 @@ jobs: secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} - - go-test: + + go-test-oss: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + runner-count: 12 + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + go-tags: "" + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + + go-test-enterprise: needs: - setup - dev-build @@ -223,6 +238,7 @@ jobs: runner-count: 12 runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -238,6 +254,7 @@ jobs: package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'" runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -253,6 +270,7 @@ jobs: go-test-flags: 'export GO_TEST_FLAGS="-short"' runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -266,6 +284,7 @@ jobs: directory: envoyextensions runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -279,6 +298,7 @@ jobs: directory: troubleshoot runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -292,6 +312,7 @@ jobs: directory: api runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -305,6 +326,7 @@ jobs: directory: api runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -318,6 +340,7 @@ jobs: directory: sdk runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -331,6 +354,7 @@ jobs: directory: sdk runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml index 5049cfaa5f37..0131582b0bef 100644 --- a/.github/workflows/reusable-unit-split.yml +++ b/.github/workflows/reusable-unit-split.yml @@ -33,6 +33,10 @@ on: repository-name: required: true type: string + go-tags: + required: false + type: string + default: "" secrets: elevated-github-token: required: true @@ -44,7 +48,7 @@ env: GOARCH: ${{inputs.go-arch}} TOTAL_RUNNERS: ${{inputs.runner-count}} CONSUL_LICENSE: ${{secrets.consul-license}} - GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + GOTAGS: ${{ inputs.go-tags}} jobs: set-test-package-matrix: diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml index b474b4c69bd3..fde340bd5edc 100644 --- a/.github/workflows/reusable-unit.yml +++ b/.github/workflows/reusable-unit.yml @@ -29,6 +29,10 @@ on: repository-name: required: true type: string + go-tags: + required: false + type: string + default: "" secrets: elevated-github-token: required: true @@ -39,7 +43,7 @@ env: GOTESTSUM_VERSION: 1.8.2 GOARCH: ${{inputs.go-arch}} CONSUL_LICENSE: ${{secrets.consul-license}} - GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + GOTAGS: ${{ inputs.go-tags}} jobs: go-test: From 3c35238da3a6243e36485b0e1757e830201700a0 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 6 Apr 2023 19:20:32 -0400 Subject: [PATCH 149/421] Backport of ci: Add success jobs. make go-test-enterprise conditional. build-distros and go-tests trigger on push to main and release branches into release/1.15.x (#16913) * backport of commit b95230eedcae50380c8f83100139fa58b7fbfa19 * backport of commit 763a5dc5d861b5f8689756feab6fa3f453281970 * backport of commit 5fd35bf7b8cc5e8294e53340c0e64abe9fc3ed0e * backport of commit c1d4fb1fd0b3795f7f8bd0881b1ff6f3d927ebf2 * backport of commit b5083fc1495f186f3fccabbb8ef4026481515782 * backport of commit 31ea1f60bc0052913a7c53aba1c57f9ed767be21 --------- Co-authored-by: John Murret --- .github/workflows/go-tests.yml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index ca90ac31d342..83c37d3b324a 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -229,6 +229,7 @@ jobs: consul-license: ${{secrets.CONSUL_LICENSE}} go-test-enterprise: + if: ${{ endsWith(github.repository, '-enterprise') }} needs: - setup - dev-build @@ -363,3 +364,47 @@ jobs: runs-on: ubuntu-latest steps: - run: "echo ok" + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + go-tests-success: + needs: + - setup + - check-generated-deep-copy + - check-generated-protobuf + - check-go-mod + - lint-consul-retry + - lint-container-test-deps + - lint-enums + - lint + - lint-32bit + # - go-test-arm64 + - go-test-enterprise + - go-test-oss + - go-test-race + - go-test-envoyextensions + - go-test-troubleshoot + - go-test-api-1-19 + - go-test-api-1-20 + - go-test-sdk-1-19 + - go-test-sdk-1-20 + - go-test-32bit + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "go-tests succeeded" From fa2dda7bb3d85d651d5e8ebfcd79e4d946c80941 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 7 Apr 2023 15:06:38 -0400 Subject: [PATCH 150/421] Backport of ci: increase ENT runner size for xl to match OSS. have build-distros use xl to match CircleCI into release/1.15.x (#16924) * no-op commit due to failed cherry-picking * increase ENT runner size for xl to match OSS. have guild-distros use xl to match CircleCI (#16920) * ci: Add success jobs. make go-test-enterprise conditional. build-distros and go-tests trigger on push to main and release branches (#16905) * Add go-tests-success job and make go-test-enterprise conditional * fixing lint-32bit reference * fixing reference to -go-test-troubleshoot * add all jobs that fan out. * fixing success job to need set up * add echo to success job * adding success jobs to build-artifacts, build-distros, and frontend. * changing the name of the job in verify ci to be consistent with other workflows * enable go-tests, build-distros, and verify-ci to run on merge to main and release branches because they currently do not with just the pull_request trigger --------- Co-authored-by: temp Co-authored-by: John Murret --- .github/scripts/get_runner_classes.sh | 3 +- .github/workflows/build-artifacts.yml | 123 ++++++++++++++++++++++++++ .github/workflows/build-distros.yml | 41 ++++++++- .github/workflows/frontend.yml | 28 ++++++ .github/workflows/go-tests.yml | 5 ++ .github/workflows/verify-ci.yml | 12 ++- 6 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-artifacts.yml diff --git a/.github/scripts/get_runner_classes.sh b/.github/scripts/get_runner_classes.sh index 4834e9e92917..b722ba768786 100755 --- a/.github/scripts/get_runner_classes.sh +++ b/.github/scripts/get_runner_classes.sh @@ -10,7 +10,8 @@ case "$GITHUB_REPOSITORY" in echo "compute-small=['self-hosted', 'linux', 'small']" >> "$GITHUB_OUTPUT" echo "compute-medium=['self-hosted', 'linux', 'medium']" >> "$GITHUB_OUTPUT" echo "compute-large=['self-hosted', 'linux', 'large']" >> "$GITHUB_OUTPUT" - echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5.2xlarge']" >> "$GITHUB_OUTPUT" + # m5d.8xlarge is equivalent to our xl custom runner in OSS + echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT" ;; *) # shellcheck disable=SC2129 diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000000..57d23d52f58d --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,123 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This workflow builds a dev binary and distributes a Docker image on every push to the main branch. +name: build-artifacts + +on: + push: + branches: + - main + +permissions: + contents: read + +env: + GOPRIVATE: github.com/hashicorp + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + dev-build-push: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; + kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; + + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: ENT specific step as we need to set elevated GitHub permissions. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Build dev binary + run: make dev + + - name: Set env vars + run: | + echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "GITHUB_BUILD_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # pin@v2.4.1 + + # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS. + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 + with: + username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} + + - name: Docker build and push + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # pin@v4.0.0 + with: + context: ./bin + file: ./build-support/docker/Consul-Dev.dockerfile + labels: COMMIT_SHA=${{ github.sha }},GITHUB_BUILD_URL=${{ env.GITHUB_BUILD_URL }} + push: true + tags: | + hashicorpdev/${{ github.event.repository.name }}:${{ env.SHORT_SHA }} + hashicorpdev/${{ github.event.repository.name }}:latest + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + build-artifacts-success: + needs: + - setup + - dev-build-push + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "build-artifacts succeeded" diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 8b6c6d9930f9..c8036a75055e 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -2,7 +2,13 @@ # It is aimed at checking new commits don't introduce any breaking build changes. name: build-distros -on: [pull_request] +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** permissions: contents: read @@ -38,7 +44,7 @@ jobs: - check-go-mod env: XC_OS: "freebsd linux windows" - runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -62,7 +68,7 @@ jobs: - check-go-mod env: XC_OS: "darwin freebsd linux solaris windows" - runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -106,3 +112,32 @@ jobs: - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + build-distros-success: + needs: + - setup + - check-go-mod + - build-386 + - build-amd64 + - build-arm + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "build-distros succeeded" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 094d86ad5555..b6451a378c36 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -104,3 +104,31 @@ jobs: - working-directory: ui/packages/consul-ui run: make test-coverage-ci + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + frontend-success: + needs: + - setup + - workspace-tests + - node-tests + - ember-build-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "frontend succeeded" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 83c37d3b324a..4d8f7b04bc42 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -10,6 +10,11 @@ on: - 'backport/docs/**' - 'backport/ui/**' - 'backport/mktg-**' + push: + branches: + # Push events on the main branch + - main + - release/** permissions: contents: read diff --git a/.github/workflows/verify-ci.yml b/.github/workflows/verify-ci.yml index 16fb4db25be5..4bd4536add1a 100644 --- a/.github/workflows/verify-ci.yml +++ b/.github/workflows/verify-ci.yml @@ -8,10 +8,16 @@ name: verify-ci permissions: contents: read -on: [pull_request] +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** jobs: - noop: + verify-ci-success: runs-on: ubuntu-latest steps: - - run: echo "ok" + - run: echo "verify-ci succeeded" From ec3df4f1b5dfc33429e94369dd6024d5f8b7d1fb Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 7 Apr 2023 16:56:15 -0400 Subject: [PATCH 151/421] Backport of docs: improve upgrade path guidance into release/1.15.x (#16927) * ISSUE_TEMPLATE: Update issue template to include ask for HCL config files for bugs (#16307) * Update bug_report.md * Fix hostname alignment checks for HTTPRoutes (#16300) * Fix hostname alignment checks for HTTPRoutes * Fix panicky xDS test flakes (#16305) * Add defensive guard to make some tests less flaky and panic less * Do the actual fix * Add stricter validation and some normalization code for API Gateway ConfigEntries (#16304) * Add stricter validation and some normalization code for API Gateway ConfigEntries * ISSUE TEMPLATE: update issue templates to include comments instead of inline text for instructions (#16313) * Update bug_report.md * Update feature_request.md * Update ui_issues.md * Update pull_request_template.md * [OSS] security: update go to 1.20.1 (#16263) * security: update go to 1.20.1 * Protobuf Refactoring for Multi-Module Cleanliness (#16302) Protobuf Refactoring for Multi-Module Cleanliness This commit includes the following: Moves all packages that were within proto/ to proto/private Rewrites imports to account for the packages being moved Adds in buf.work.yaml to enable buf workspaces Names the proto-public buf module so that we can override the Go package imports within proto/buf.yaml Bumps the buf version dependency to 1.14.0 (I was trying out the version to see if it would get around an issue - it didn't but it also doesn't break things and it seemed best to keep up with the toolchain changes) Why: In the future we will need to consume other protobuf dependencies such as the Google HTTP annotations for openapi generation or grpc-gateway usage. There were some recent changes to have our own ratelimiting annotations. The two combined were not working when I was trying to use them together (attempting to rebase another branch) Buf workspaces should be the solution to the problem Buf workspaces means that each module will have generated Go code that embeds proto file names relative to the proto dir and not the top level repo root. This resulted in proto file name conflicts in the Go global protobuf type registry. The solution to that was to add in a private/ directory into the path within the proto/ directory. That then required rewriting all the imports. Is this safe? AFAICT yes The gRPC wire protocol doesn't seem to care about the proto file names (although the Go grpc code does tack on the proto file name as Metadata in the ServiceDesc) Other than imports, there were no changes to any generated code as a result of this. * new docs for consul and consul-k8s troubleshoot command (#16284) * new docs for consul and consul-k8s troubleshoot command * add changelog * add troubleshoot command * address comments, and update cli output to match * revert changes to troubleshoot upstreams, changes will happen in separate pr * Update .changelog/16284.txt Co-authored-by: Nitya Dhanushkodi * address comments * update trouble proxy output * add missing s, add required fields in usage --------- Co-authored-by: Nitya Dhanushkodi * Normalize all API Gateway references (#16316) * Fix HTTPRoute and TCPRoute expectation for enterprise metadata (#16322) * ISSUE_TEMPLATE: formatting for comments (#16325) * Update all templates. * fix: revert go mod compat for sdk,api to 1.19 (#16323) * fix: add tls config to unix socket when https is used (#16301) * fix: add tls config to unix socket when https is used * unit test and changelog * fix flakieness (#16338) * chore: document and unit test sdk/testutil/retry (#16049) * [API Gateway] Validate listener name is not empty (#16340) * [API Gateway] Validate listener name is not empty * Update docstrings and test * Fix issue with peer services incorrectly appearing as connect-enabled. (#16339) Prior to this commit, all peer services were transmitted as connect-enabled as long as a one or more mesh-gateways were healthy. With this change, there is now a difference between typical services and connect services transmitted via peering. A service will be reported as "connect-enabled" as long as any of these conditions are met: 1. a connect-proxy sidecar is registered for the service name. 2. a connect-native instance of the service is registered. 3. a service resolver / splitter / router is registered for the service name. 4. a terminating gateway has registered the service. * [API Gateway] Turn down controller log levels (#16348) * [API Gateway] Fix targeting service splitters in HTTPRoutes (#16350) * [API Gateway] Fix targeting service splitters in HTTPRoutes * Fix test description * [API Gateway] Various fixes for Config Entry fields (#16347) * [API Gateway] Various fixes for Config Entry fields * simplify logic per PR review * upgrade test: splitter and resolver config entry in peered cluster (#16356) * Upgrade Alpine image to 3.17 (#16358) * Update existing docs from Consul API Gateway -> API Gateway for Kubernetes (#16360) * Update existing docs from Consul API Gateway -> API Gateway for Kubernetes * Update page header to reflect page title change * Update nav title to match new page title * initial code (#16296) * Add changelog entry for API Gateway (Beta) (#16369) * Placeholder commit for changelog entry * Add changelog entry announcing support for API Gateway on VMs * Adjust casing * [API Gateway] Fix infinite loop in controller and binding non-accepted routes and gateways (#16377) * Rate limiter/add ip prefix (#16342) * add support for prefixes in the config tree * fix to use default config when the prefix have no config * Documentation update: Adding K8S clusters to external Consul servers (#16285) * Remove Consul Client installation option With Consul-K8S 1.0 and introduction of Consul-Dataplane, K8S has the option to run without running Consul Client agents. * remove note referring to the same documentation * Added instructions on the use of httpsPort when servers are not running TLS enabled * Modified titile and description * Add docs for usage endpoint and command (#16258) * Add docs for usage endpoint and command * NET-2285: Assert total number of expected instances by Consul (#16371) * set BRANCH_NAME to release-1.15.x (#16374) * Docs/rate limiting 1.15 (#16345) * Added rate limit section to agent overview, updated headings per style guide * added GTRL section and overview * added usage docs for rate limiting 1.15 * added file for initializing rate limits * added steps for initializing rate limits * updated descriptions for rate_limits in agent conf * updated rate limiter-related metrics * tweaks to agent index * Apply suggestions from code review Co-authored-by: Dhia Ayachi Co-authored-by: Krastin Krastev * Apply suggestions from code review Co-authored-by: Krastin Krastev * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --------- Co-authored-by: Dhia Ayachi Co-authored-by: Krastin Krastev Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * [UI] CC-4031: change from Action, a and button to hds::Button (#16251) * Correct WAL metrics registrations (#16388) * chore: remove stable-website (#16386) * Refactor the disco chain -> xds logic (#16392) * Add envoy extension docs (#16376) * Add envoy extension docs * Update message about envoy extensions with proxy defaults * fix tab error * Update website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx * fix operator prerender issue * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * update envoyextension warning in proxy defaults so its inline * Update website/content/docs/connect/proxies/envoy-extensions/index.mdx --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * upgrade test: peering with resolver and failover (#16391) * Troubleshoot service to service comms (#16385) * Troubleshoot service to service comms * adjustments * breaking fix * api-docs breaking fix * Links added to CLI pages * Update website/content/docs/troubleshoot/troubleshoot-services.mdx Co-authored-by: Eric Haberkorn * Update website/content/docs/troubleshoot/troubleshoot-services.mdx Co-authored-by: Tu Nguyen * Update website/content/docs/troubleshoot/troubleshoot-services.mdx Co-authored-by: Tu Nguyen * nav re-ordering * Edits recommended in code review --------- Co-authored-by: Eric Haberkorn Co-authored-by: Tu Nguyen * Docs/cluster peering 1.15 updates (#16291) * initial commit * initial commit * Overview updates * Overview page improvements * More Overview improvements * improvements * Small fixes/updates * Updates * Overview updates * Nav data * More nav updates * Fix * updates * Updates + tip test * Directory test * refining * Create restructure w/ k8s * Single usage page * Technical Specification * k8s pages * typo * L7 traffic management * Manage connections * k8s page fix * Create page tab corrections * link to k8s * intentions * corrections * Add-on intention descriptions * adjustments * Missing * Diagram improvements * Final diagram update * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu * diagram name fix * Fixes * Updates to index.mdx * Tech specs page corrections * Tech specs page rename * update link to tech specs * K8s - new pages + tech specs * k8s - manage peering connections * k8s L7 traffic management * Separated establish connection pages * Directory fixes * Usage clean up * k8s docs edits * Updated nav data * CodeBlock Component fix * filename * CodeBlockConfig removal * Redirects * Update k8s filenames * Reshuffle k8s tech specs for clarity, fmt yaml files * Update general cluster peering docs, reorder CLI > API > UI, cross link to kubernetes * Fix config rendering in k8s usage docs, cross link to general usage from k8s docs * fix legacy link * update k8s docs * fix nested list rendering * redirect fix * page error --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: David Yu Co-authored-by: Tu Nguyen * Fix rendering error on new operator usage docs (#16393) * add missing field to oss struct (#16401) * fix(docs): correct rate limit metrics (#16400) * Fix various flaky tests (#16396) * Native API Gateway Docs (#16365) * Create empty files * Copy over content for overview * Copy over content for usage * Copy over content for api-gateway config * Copy over content for http-route config * Copy over content for tcp-route config * Copy over content for inline-certificate config * Add docs to the sidebar * Clean up overview. Start cleaning up usage * Add BETA badge to API Gateways portion of nav * Fix header * Fix up usage * Fix up API Gateway config * Update paths to be consistent w/ other gateway docs * Fix up http-route * Fix up inline-certificate * rename path * Fix up tcp-route * Add CodeTabs * Add headers to config pages * Fix configuration model for http route and inline certificate * Add version callout to API gateway overview page * Fix values for inline certificate * Fix values for api gateway configuration * Fix values for TCP Route config * Fix values for HTTP Route config * Adds link from k8s gateway to vm gateway page * Remove versioning warning * Serve overview page at ../api-gateway, consistent w/ mesh-gateway * Remove weight field from tcp-route docs * Linking to usage instead of overview from k8s api-gateway to vm api-gateway * Fix issues in usage page * Fix links in usage * Capitalize Kubernetes * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * remove optional callout * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Apply suggestions from code review * Update website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx * Fix formatting of Hostnames * Update website/content/docs/api-gateway/index.mdx * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: Andrew Stucki * Add cross-linking of config entries * Fix rendering error on new operator usage docs * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Apply suggestions from code review * Apply suggestions from code review * Add BETA badges to config entry links * http route updates * Add Enterprise keys * Use map instead of list for meta field, use consistent formatting * Convert spaces to tabs * Add all Enterprise info to TCP Route * Use pascal case for JSON api-gateway example * Add enterprise to HCL api-gw cfg * Use pascal case for missed JSON config fields * Add enterprise to JSON api-gw cfg * Add enterprise to api-gw values * adds enterprise to http route * Update website/content/docs/connect/gateways/api-gateway/index.mdx Co-authored-by: danielehc <40759828+danielehc@users.noreply.github.com> * Add enterprise to api-gw spec * Add missing namespace, partition + meta to specification * fixes for http route * Fix ordering of API Gatetway cfg spec items * whitespace * Add linking of values to tcp * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * Fix comma in wrong place * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * Move Certificates down * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * Tabs to spaces in httproute * Use configuration entry instead of config entry * Fix indentations on api-gateway and tcp-route * Add whitespace between code block and prose * Apply suggestions from code review Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * adds <> to http route --------- Co-authored-by: Nathan Coleman Co-authored-by: Melisa Griffin Co-authored-by: Tu Nguyen Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Tu Nguyen Co-authored-by: Melisa Griffin Co-authored-by: Andrew Stucki Co-authored-by: danielehc <40759828+danielehc@users.noreply.github.com> Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * NET-2286: Add tests to verify traffic redirects between services (#16390) * Try DRYing up createCluster in integration tests (#16199) * add back staging bits (#16411) * Fix a couple inconsistencies in `operator usage instances` command (#16260) * NO_JIRA: refactor validate function in traffic mgt tests (#16422) * Basic gobased API gateway spinup test (#16278) * wip, proof of concept, gateway service being registered, don't know how to hit it * checkpoint * Fix up API Gateway go tests (#16297) * checkpoint, getting InvalidDiscoveryChain route protocol does not match targeted service protocol * checkpoint * httproute hittable * tests working, one header test failing * differentiate services by status code, minor cleanup * working tests * updated GetPort interface * fix getport --------- Co-authored-by: Andrew Stucki * Fix attempt for test fail panics in xDS (#16319) * Fix attempt for test fail panics in xDS * switch to a mutex pointer * update changelog (#16426) * update changelog * fix changelog formatting * feat: update alerts to Hds::Alert component (CC-4035) (#16412) * fix: ui tests run is fixed (applying class attribute twice to the hbs element caused the issue (#16428) * Refactor and move wal docs (#16387) * Add WAL documentation. Also fix some minor metrics registration details * Add tests to verify metrics are registered correctly * refactor and move wal docs * Updates to the WAL overview page * updates to enable WAL usage topic * updates to the monitoring WAL backend topic * updates for revert WAL topic * a few tweaks to overview and udpated metadescriptions * Apply suggestions from code review Co-authored-by: Paul Banks * make revert docs consistent with enable * Apply suggestions from code review Co-authored-by: Paul Banks * address feedback * address final feedback * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --------- Co-authored-by: Paul Banks Co-authored-by: trujillo-adam Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * UI: Update Consul UI colors to use HDS colors (#16111) * update red color variables to hds * change background red to be one step lighter * map oranges * map greens * map blues * map greys * delete themes, colours: lemon, magenta, strawberry, and vault color aliases * add unmapped rainbow colours * replace white and transparent vars, remove unused semantic vars and frame placeholders * small tweaks to improve contrast, change node health status x/check colours for non-voters to match design doc, replace semantic colour action w hds colour * add unmapped grays, remove dark theme, manually set nav bar to use dark colours * map consul pink colour * map yellows * add unmapped oranges, delete light theme * remove readme, base variables, clean up dangling colours * Start working on the nav disclosure menus * Update main-nav-horizontal dropdowns * Format template * Update box-shadow tokens * Replace --tone- usage with tokens * Update nav disabled state and panel border colour * Replace rgb usage on tile * Fix permissions modal overlay * More fixes * Replace orange-500 with amber-200 * Update badge colors * Update vertical sidebar colors * Remove top border on consul peer list ul --------- Co-authored-by: wenincode * Add missing link (#16437) * docs: remove extra whitespace in frontmatter (#16436) * Delete Vagrantfile (#16442) * upgrade test: consolidate resolver test cases (#16443) * UI: Fix rendering issue in search and lists (#16444) * Upgrade ember-cli-string-helpers * add extra lock change * Update docs for consul-k8s 1.1.0 (#16447) * Update ingress-gateways.mdx (#16330) * Update ingress-gateways.mdx Added an example of running the HELM install for the ingress gateways using values.yaml * Apply suggestions from code review * Update ingress-gateways.mdx Adds closing back ticks on example command. The suggesting UI strips them out. --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * grpc: fix data race in balancer registration (#16229) Registering gRPC balancers is thread-unsafe because they are stored in a global map variable that is accessed without holding a lock. Therefore, it's expected that balancers are registered _once_ at the beginning of your program (e.g. in a package `init` function) and certainly not after you've started dialing connections, etc. > NOTE: this function must only be called during initialization time > (i.e. in an init() function), and is not thread-safe. While this is fine for us in production, it's challenging for tests that spin up multiple agents in-memory. We currently register a balancer per- agent which holds agent-specific state that cannot safely be shared. This commit introduces our own registry that _is_ thread-safe, and implements the Builder interface such that we can call gRPC's `Register` method once, on start-up. It uses the same pattern as our resolver registry where we use the dial target's host (aka "authority"), which is unique per-agent, to determine which builder to use. * cli: ensure acl token read -self works (#16445) Fixes a regression in #16044 The consul acl token read -self cli command should not require an -accessor-id because typically the persona invoking this would not already know the accessor id of their own token. * docs: Add backwards compatibility for Consul 1.14.x and consul-dataplane in the Envoy compat matrix (#16462) * Update envoy.mdx * gateways: add e2e test for API Gateway HTTPRoute ParentRef change (#16408) * test(gateways): add API Gateway HTTPRoute ParentRef change test * test(gateways): add checkRouteError helper * test(gateways): remove EOF check in CI this seems to sometimes be 'connection reset by peer' instead * Update test/integration/consul-container/test/gateways/http_route_test.go * Gateway Test HTTPPathRewrite (#16418) * add http url path rewrite * add Mike's test back in * update kind to use api.APIGateway * cli: remove stray whitespace when loading the consul version from the VERSION file (#16467) Fixes a regression from #15631 in the output of `consul version` from: Consul v1.16.0-dev +ent Revision 56b86acbe5+CHANGES to Consul v1.16.0-dev+ent Revision 56b86acbe5+CHANGES * Docs/services refactor docs day 122022 (#16103) * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * fixed cross-links between new topics * updated links to the new services pages * fixed bad links in scale file * tweaks to titles and phrasing * fixed typo in checks.mdx * started updating the conf ref to latest template * update SD conf ref to match latest CT standard * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> * remove previous version of the checks page * fixed cross-links * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> --------- Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> * docs: clarify license expiration upgrade behavior (#16464) * add provider ca auth-method support for azure Does the required dance with the local HTTP endpoint to get the required data for the jwt based auth setup in Azure. Keeps support for 'legacy' mode where all login data is passed on via the auth methods parameters. Refactored check for hardcoded /login fields. * Changed titles for services pages to sentence style cap (#16477) * Changed titles for services pages to sentence style cap * missed a meta title * docs: Consul 1.15.0 and Consul K8s 1.0 release notes (#16481) * add new release notes --------- Co-authored-by: Tu Nguyen * fix (cli): return error msg if acl policy not found (#16485) * fix: return error msg if acl policy not found * changelog * add test * update services nav titles (#16484) * Improve ux to help users avoid overwriting fields of ACL tokens, roles and policies (#16288) * Deprecate merge-policies and add options add-policy-name/add-policy-id to improve CLI token update command * deprecate merge-roles fields * Fix potential flakey tests and update ux to remove 'completely' + typo fixes * NET-2292: port ingress-gateway test case "http" from BATS addendum (#16490) * docs: Update release notes with Envoy compat issue (#16494) * Update v1_15_x.mdx --------- Co-authored-by: Tu Nguyen * Suppress AlreadyRegisteredError to fix test retries (#16501) * Suppress AlreadyRegisteredError to fix test retries * Remove duplicate sink * Speed up test by registering services concurrently (#16509) * add provider ca support for jwt file base auth Adds support for a jwt token in a file. Simply reads the file and sends the read in jwt along to the vault login. It also supports a legacy mode with the jwt string being passed directly. In which case the path is made optional. * docs(architecture): remove merge conflict leftovers (#16507) * add provider ca auth support for kubernetes Adds support for Kubernetes jwt/token file based auth. Only needs to read the file and save the contents as the jwt/token. * Merge pull request #4538 from hashicorp/NET-2396 (#16516) NET-2396: refactor test to reduce duplication * Merge pull request #4584 from hashicorp/refactor_cluster_config (#16517) NET-2841: PART 1 - refactor NewPeeringCluster to support custom config * Add ServiceResolver RequestTimeout for route timeouts to make TerminatingGateway upstream timeouts configurable (#16495) * Leverage ServiceResolver ConnectTimeout for route timeouts to make TerminatingGateway upstream timeouts configurable * Regenerate golden files * Add RequestTimeout field * Add changelog entry * Fix issue where terminating gateway service resolvers weren't properly cleaned up (#16498) * Fix issue where terminating gateway service resolvers weren't properly cleaned up * Add integration test for cleaning up resolvers * Add changelog entry * Use state test and drop integration test * Add support for failover policies (#16505) * modified unsupported envoy version error (#16518) - When an envoy version is out of a supported range, we now return the envoy version being used as `major.minor.x` to indicate that it is the minor version at most that is incompatible - When an envoy version is in the list of unsupported envoy versions we return back the envoy version in the error message as `major.minor.patch` as now the exact version matters. * Remove private prefix from proto-gen-rpc-glue e2e test (#16433) * Fix resolution of service resolvers with subsets for external upstreams (#16499) * Fix resolution of service resolvers with subsets for external upstreams * Add tests * Add changelog entry * Update view filter logic * fixed broken links associated with cluster peering updates (#16523) * fixed broken links associated with cluster peering updates * additional links to fix * typos * fixed redirect file * add provider ca support for approle auth-method Adds support for the approle auth-method. Only handles using the approle role/secret to auth and it doesn't support the agent's extra management configuration options (wrap and delete after read) as they are not required as part of the auth (ie. they are vault agent things). * update connect/ca's vault AuthMethod conf section (#16346) Updated Params field to re-frame as supporting arguments specific to the supported vault-agent auth-auth methods with links to each methods "#configuration" section. Included a call out limits on parameters supported. * proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher (#16497) Receiving an "acl not found" error from an RPC in the agent cache and the streaming/event components will cause any request loops to cease under the assumption that they will never work again if the token was destroyed. This prevents log spam (#14144, #9738). Unfortunately due to things like: - authz requests going to stale servers that may not have witnessed the token creation yet - authz requests in a secondary datacenter happening before the tokens get replicated to that datacenter - authz requests from a primary TO a secondary datacenter happening before the tokens get replicated to that datacenter The caller will get an "acl not found" *before* the token exists, rather than just after. The machinery added above in the linked PRs will kick in and prevent the request loop from looping around again once the tokens actually exist. For `consul-dataplane` usages, where xDS is served by the Consul servers rather than the clients ultimately this is not a problem because in that scenario the `agent/proxycfg` machinery is on-demand and launched by a new xDS stream needing data for a specific service in the catalog. If the watching goroutines are terminated it ripples down and terminates the xDS stream, which CDP will eventually re-establish and restart everything. For Consul client usages, the `agent/proxycfg` machinery is ahead-of-time launched at service registration time (called "local" in some of the proxycfg machinery) so when the xDS stream comes in the data is already ready to go. If the watching goroutines terminate it should terminate the xDS stream, but there's no mechanism to re-spawn the watching goroutines. If the xDS stream reconnects it will see no `ConfigSnapshot` and will not get one again until the client agent is restarted, or the service is re-registered with something changed in it. This PR fixes a few things in the machinery: - there was an inadvertent deadlock in fetching snapshot from the proxycfg machinery by xDS, such that when the watching goroutine terminated the snapshots would never be fetched. This caused some of the xDS machinery to get indefinitely paused and not finish the teardown properly. - Every 30s we now attempt to re-insert all locally registered services into the proxycfg machinery. - When services are re-inserted into the proxycfg machinery we special case "dead" ones such that we unilaterally replace them rather that doing that conditionally. * NET-2903 Normalize weight for http routes (#16512) * NET-2903 Normalize weight for http routes * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Add some basic UI improvements for api-gateway services (#16508) * Add some basic ui improvements for api-gateway services * Add changelog entry * Use ternary for null check * Update gateway doc links * rename changelog entry for new PR * Fix test * fixes empty link in DNS usage page (#16534) * NET-2904 Fixes API Gateway Route Service Weight Division Error * Improve ux around ACL token to help users avoid overwriting node/service identities (#16506) * Deprecate merge-node-identities and merge-service-identities flags * added tests for node identities changes * added changelog file and docs * Follow-up fixes to consul connect envoy command (#16530) * Merge pull request #4573 from hashicorp/NET-2841 (#16544) * Merge pull request #4573 from hashicorp/NET-2841 NET-2841: PART 2 refactor upgrade tests to include version 1.15 * update upgrade versions * upgrade test: discovery chain across partition (#16543) * Update the consul-k8s cli docs for the new `proxy log` subcommand (#16458) * Update the consul-k8s cli docs for the new `proxy log` subcommand * Updated consul-k8s docs from PR feedback * Added proxy log command to release notes * Delete test-link-rewrites.yml (#16546) * feat: update notification to use hds toast component (#16519) * Fix flakey tests related to ACL token updates (#16545) * Fix flakey tests related to ACL token updates * update all acl token update tests * extra create_token function to its own thing * support vault auth config for alicloud ca provider Add support for using existing vault auto-auth configurations as the provider configuration when using Vault's CA provider with AliCloud. AliCloud requires 2 extra fields to enable it to use STS (it's preferred auth setup). Our vault-plugin-auth-alicloud package contained a method to help generate them as they require you to make an http call to a faked endpoint proxy to get them (url and headers base64 encoded). * Update docs to reflect functionality (#16549) * Update docs to reflect functionality * make consistent with other client runtimes * upgrade test: use retry with ModifyIndex and remove ent test file (#16553) * add agent locality and replicate it across peer streams (#16522) * docs: Document config entry permissions (#16556) * Broken link fixes (#16566) * NET-2954: Improve integration tests CI execution time (#16565) * NET-2954: Improve integration tests CI execution time * fix ci * remove comments and modify config file * fix bug that can lead to peering service deletes impacting the state of local services (#16570) * Update changelog with patch releases (#16576) * Bump submodules from latest 1.15.1 patch release (#16578) * Update changelog with Consul patch releases 1.13.7, 1.14.5, 1.15.1 * Bump submodules from latest patch release * Forgot one * website: adds content-check command and README update (#16579) * added a backport-checker GitHub action (#16567) * added a backport-checker GitHub action * Update .github/workflows/backport-checker.yml * auto-updated agent/uiserver/dist/ from commit 63204b518 (#16587) Co-authored-by: hc-github-team-consul-core * GRPC stub for the ResourceService (#16528) * UI: Fix htmlsafe errors throughout the app (#16574) * Upgrade ember-intl * Add changelog * Add yarn lock * Add namespace file with build tag for OSS gateway tests (#16590) * Add namespace file with build tag for OSS tests * Remove TODO comment * JIRA pr check: Filter out OSS/ENT merges (#16593) * jira pr check filter out dependabot and oss/ent merges * allow setting locality on services and nodes (#16581) * Add Peer Locality to Discovery Chains (#16588) Add peer locality to discovery chains * fixes for unsupported partitions field in CRD metadata block (#16604) * fixes for unsupported partitions field in CRD metadata block * Apply suggestions from code review Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --------- Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> * Create a weekly 404 checker for all Consul docs content (#16603) * Consul WAN Fed with Vault Secrets Backend document updates (#16597) * Consul WAN Fed with Vault Secrets Backend document updates * Corrected dc1-consul.yaml and dc2-consul.yaml file highlights * Update website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Allow HCP metrics collection for Envoy proxies Co-authored-by: Ashvitha Sridharan Co-authored-by: Freddy Add a new envoy flag: "envoy_hcp_metrics_bind_socket_dir", a directory where a unix socket will be created with the name `_.sock` to forward Envoy metrics. If set, this will configure: - In bootstrap configuration a local stats_sink and static cluster. These will forward metrics to a loopback listener sent over xDS. - A dynamic listener listening at the socket path that the previously defined static cluster is sending metrics to. - A dynamic cluster that will forward traffic received at this listener to the hcp-metrics-collector service. Reasons for having a static cluster pointing at a dynamic listener: - We want to secure the metrics stream using TLS, but the stats sink can only be defined in bootstrap config. With dynamic listeners/clusters we can use the proxy's leaf certificate issued by the Connect CA, which isn't available at bootstrap time. - We want to intelligently route to the HCP collector. Configuring its addreess at bootstrap time limits our flexibility routing-wise. More on this below. Reasons for defining the collector as an upstream in `proxycfg`: - The HCP collector will be deployed as a mesh service. - Certificate management is taken care of, as mentioned above. - Service discovery and routing logic is automatically taken care of, meaning that no code changes are required in the xds package. - Custom routing rules can be added for the collector using discovery chain config entries. Initially the collector is expected to be deployed to each admin partition, but in the future could be deployed centrally in the default partition. These config entries could even be managed by HCP itself. * Add copywrite setup file (#16602) * Add sameness-group configuration entry. (#16608) This commit adds a sameness-group config entry to the API and structs packages. It includes some validation logic and a new memdb index that tracks the default sameness-group for each partition. Sameness groups will simplify the effort of managing failovers / intentions / exports for peers and partitions. Note that this change purely to introduce the configuration entry and does not include the full functionality of sameness-groups. * Preserve CARoots when updating Vault CA configuration (#16592) If a CA config update did not cause a root change, the codepath would return early and skip some steps which preserve its intermediate certificates and signing key ID. This commit re-orders some code and prevents updates from generating new intermediate certificates. * Add UI copyright headers files (#16614) * Add copyright headers to UI files * Ensure copywrite file ignores external libs * Docs discovery typo (#16628) * docs(discovery): typo * docs(discovery): EOF and trim lines --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Fix issue with trust bundle read ACL check. (#16630) This commit fixes an issue where trust bundles could not be read by services in a non-default namespace, unless they had excessive ACL permissions given to them. Prior to this change, `service:write` was required in the default namespace in order to read the trust bundle. Now, `service:write` to a service in any namespace is sufficient. * Basic resource type registry (#16622) * Backport ENT-4704 (#16612) * feat: update typography to consume hds styles (#16577) * Add known issues to Raft WAL docs. (#16600) * Add known issues to Raft WAL docs. * Refactor update based on review feedback * Tune 404 checker to exclude false-positives and use intended file path (#16636) * Update e2e tests for namespaces (#16627) * Refactored "NewGatewayService" to handle namespaces, fixed TestHTTPRouteFlattening test * Fixed existing http_route tests for namespacing * Squash aclEnterpriseMeta for ResourceRefs and HTTPServices, accept namespace for creating connect services and regular services * Use require instead of assert after creating namespaces in http_route_tests * Refactor NewConnectService and NewGatewayService functions to use cfg objects to reduce number of method args * Rename field on SidecarConfig in tests from `SidecarServiceName` to `Name` to avoid stutter * net 2731 ip config entry OSS version (#16642) * ip config entry * name changing * move to ent * ent version * renaming * change format * renaming * refactor * add default values * fix confusing spiffe ids in golden tests (#16643) * First cluster grpc service should be NodePort for the second cluster to connect (#16430) * First cluster grpc service should be NodePort This is based on the issue opened here https://github.com/hashicorp/consul-k8s/issues/1903 If you follow the documentation https://developer.hashicorp.com/consul/docs/k8s/deployment-configurations/single-dc-multi-k8s exactly as it is, the first cluster will only create the consul UI service on NodePort but not the rest of the services (including for grpc). By default, from the helm chart, they are created as headless services by setting clusterIP None. This will cause an issue for the second cluster to discover consul server on the first cluster over gRPC as it cannot simply cannot through gRPC default port 8502 and it ends up in an error as shown in the issue https://github.com/hashicorp/consul-k8s/issues/1903 As a solution, the grpc service should be exposed using NodePort (or LoadBalancer). I added those changes required in both cluster1-values.yaml and cluster2-values.yaml, and also a description for those changes for the normal users to understand. Kindly review and I hope this PR will be accepted. * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Add in query options for catalog service existing in a specific (#16652) namespace when creating service for tests * fix: add AccessorID property to PUT token request (#16660) * add sameness group support to service resolver failover and redirects (#16664) * Fix incorrect links on Envoy extensions documentation (#16666) * [API Gateway] Fix invalid cluster causing gateway programming delay (#16661) * Add test for http routes * Add fix * Fix tests * Add changelog entry * Refactor and fix flaky tests * Bump tomhjp/gh-action-jira-search from 0.2.1 to 0.2.2 (#16667) Bumps [tomhjp/gh-action-jira-search](https://github.com/tomhjp/gh-action-jira-search) from 0.2.1 to 0.2.2. - [Release notes](https://github.com/tomhjp/gh-action-jira-search/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-search/compare/v0.2.1...v0.2.2) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-search dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump atlassian/gajira-transition from 2.0.1 to 3.0.1 (#15921) Bumps [atlassian/gajira-transition](https://github.com/atlassian/gajira-transition) from 2.0.1 to 3.0.1. - [Release notes](https://github.com/atlassian/gajira-transition/releases) - [Commits](https://github.com/atlassian/gajira-transition/compare/v2.0.1...v3.0.1) --- updated-dependencies: - dependency-name: atlassian/gajira-transition dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * Snapshot restore tests (#16647) * add snapshot restore test * add logstore as test parameter * Use the correct image version * make sure we read the logs from a followers to test the follower snapshot install path. * update to raf-wal v0.3.0 * add changelog. * updating changelog for bug description and removed integration test. * setting up test container builder to only set logStore for 1.15 and higher --------- Co-authored-by: Paul Banks Co-authored-by: John Murret * add sameness groups to discovery chains (#16671) * feat: add category annotation to RPC and gRPC methods (#16646) * Update GH actions to create Jira issue automatically (#16656) * Adds check to verify that the API Gateway is being created with at least one listener * Fix route subscription when using namespaces (#16677) * Fix route subscription when using namespaces * Update changelog * Fix changelog entry to reference that the bug was enterprise only * peering: peering partition failover fixes (#16673) add local source partition for peered upstreams * fix jira sync actions, remove custom fields (#16686) * Docs/update jira sync pr issue (#16688) * fix jira sync actions, remove custom fields * remove more additional fields, debug * Docs: Jira sync Update issuetype to bug (#16689) * update issuetype to bug * fix conditional for pr edu * build(deps): bump tomhjp/gh-action-jira-create from 0.2.0 to 0.2.1 (#16685) Bumps [tomhjp/gh-action-jira-create](https://github.com/tomhjp/gh-action-jira-create) from 0.2.0 to 0.2.1. - [Release notes](https://github.com/tomhjp/gh-action-jira-create/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-create/compare/v0.2.0...v0.2.1) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-create dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * build(deps): bump tomhjp/gh-action-jira-comment from 0.1.0 to 0.2.0 (#16684) Bumps [tomhjp/gh-action-jira-comment](https://github.com/tomhjp/gh-action-jira-comment) from 0.1.0 to 0.2.0. - [Release notes](https://github.com/tomhjp/gh-action-jira-comment/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-comment/compare/v0.1.0...v0.2.0) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-comment dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * NET-2397: Add readme.md to upgrade test subdirectory (#16610) * NET-2397: Add readme.md to upgrade test subdirectory * remove test code * fix link and update steps of adding new test cases (#16654) * fix link and update steps of adding new test cases * Apply suggestions from code review Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> --------- Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> --------- Co-authored-by: cskh Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> * chore: replace hardcoded node name with a constant (#16692) * Fix broken links from api docs (#16695) * Update WAL Known issues (#16676) * UI: update Ember to 3.28.6 (#16616) --------- Co-authored-by: wenincode * Regen helm docs (#16701) * Remove unused are hosts set check (#16691) * Remove unused are hosts set check * Remove all traces of unused 'AreHostsSet' parameter * Remove unused Hosts attribute * Remove commented out use of snap.APIGateway.Hosts * [NET-3029] Migrate build-distros to GHA (#16669) * migrate build distros to GHA Signed-off-by: Dan Bond * build-arm Signed-off-by: Dan Bond * don't use matrix Signed-off-by: Dan Bond * check-go-mod Signed-off-by: Dan Bond * add notify slack script Signed-off-by: Dan Bond * notify slack if failure Signed-off-by: Dan Bond * rm notify slack script Signed-off-by: Dan Bond * fix check-go-mod job Signed-off-by: Dan Bond --------- Signed-off-by: Dan Bond * Update envoy extension docs, service-defaults, add multi-config example for lua (#16710) * fix build workflow (#16719) Signed-off-by: Dan Bond * Helm docs without developer.hashicorp.com prefix (#16711) This was causing linter errors * add extra resiliency to snapshot restore test (#16712) * fix: gracefully fail on invalid port number (#16721) * Copyright headers for config files git + circleci (#16703) * Copyright headers for config files git + circleci * Release folder copyright headers * fix bug where pqs that failover to a cluster peer dont un-fail over (#16729) * add enterprise xds tests (#16738) * delete config when nil (#16690) * delete config when nil * fix mock interface implementation * fix handler test to use the right assertion * extract DeleteConfig as a separate API. * fix mock limiter implementation to satisfy the new interface * fix failing tests * add test comments * Changelog for audit logging fix. (#16700) * Changelog for audit logging fix. * Use GH issues type for edu board (#16750) * fix: remove unused tenancy category from rate limit spec (#16740) * Remove version bump from CRT workflow (#16728) This bumps the version to reflect the next patch release; however, we use a specific branch for each patch release and so never wind up cutting a release directly from the `release/1.15.x` (for example) where this is intended to work. * tests instantiating clients w/o shutting down (#16755) noticed via their port still in use messages. * RELENG-471: Remove obsolete load-test workflow (#16737) * Remove obsolete load-test workflow * remove load-tests from circleci config. --------- Co-authored-by: John Murret * add failover policy to ProxyConfigEntry in api (#16759) * add failover policy to ProxyConfigEntry in api * update docs * Fix broken links in Consul docs (#16640) * Fix broken links in Consul docs * more broken link fixes * more 404 fixes * 404 fixes * broken link fix --------- Co-authored-by: Tu Nguyen * Change partition for peers in discovery chain targets (#16769) This commit swaps the partition field to the local partition for discovery chains targeting peers. Prior to this change, peer upstreams would always use a value of default regardless of which partition they exist in. This caused several issues in xds / proxycfg because of id mismatches. Some prior fixes were made to deal with one-off id mismatches that this PR also cleans up, since they are no longer needed. * Docs/intentions refactor docs day 2022 (#16758) * converted intentions conf entry to ref CT format * set up intentions nav * add page for intentions usage * final intentions usage page * final intentions overview page * fixed old relative links * updated diagram for overview * updated links to intentions content * fixed typo in updated links * rename intentions overview page file to index * rollback link updates to intentions overview * fixed nav * Updated custom HTML in API and CLI pages to MD * applied suggestions from review to index page * moved conf examples from usage to conf ref * missed custom HTML section * applied additional feedback * Apply suggestions from code review Co-authored-by: Tu Nguyen * updated headings in usage page * renamed files and udpated nav * updated links to new file names * added redirects and final tweaks * typo --------- Co-authored-by: Tu Nguyen * Add storage backend interface and in-memory implementation (#16538) Introduces `storage.Backend`, which will serve as the interface between the Resource Service and the underlying storage system (Raft today, but in the future, who knows!). The primary design goal of this interface is to keep its surface area small, and push as much functionality as possible into the layers above, so that new implementations can be added with little effort, and easily proven to be correct. To that end, we also provide a suite of "conformance" tests that can be run against a backend implementation to check it behaves correctly. In this commit, we introduce an initial in-memory storage backend, which is suitable for tests and when running Consul in development mode. This backend is a thin wrapper around the `Store` type, which implements a resource database using go-memdb and our internal pub/sub system. `Store` will also be used to handle reads in our Raft backend, and in the future, used as a local cache for external storage systems. * Fix bug in changelog checker where bash variable is not quoted (#16681) * Read(...) endpoint for the resource service (#16655) * Fix Edu Jira automation (#16778) * Fix struct tags for TCPService enterprise meta (#16781) * Fix struct tags for TCPService enterprise meta * Add changelog * Expand route flattening test for multiple namespaces (#16745) * Exand route flattening test for multiple namespaces * Add helper for checking http route config entry exists without checking for bound status * Fix port and hostname check for http route flattening test * WatchList(..) endpoint for the resource service (#16726) * Allocate virtual ip for resolver/router/splitter config entries (#16760) * add ip rate limiter controller OSS parts (#16790) * Resource service List(..) endpoint (#16753) * changes to support new PQ enterprise fields (#16793) * add scripts for testing locally consul-ui-toolkit (#16794) * Update normalization of route refs (#16789) * Use merge of enterprise meta's rather than new custom method * Add merge logic for tcp routes * Add changelog * Normalize certificate refs on gateways * Fix infinite call loop * Explicitly call enterprise meta * copyright headers for agent folder (#16704) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * Copyright headers for command folder (#16705) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * copyright headers for agent folder * Copyright headers for command folder * fix merge conflicts * Add copyright headers for acl, api and bench folders (#16706) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * copyright headers for agent folder * fix merge conflicts * copyright headers for agent folder * Ignore test data files * fix proto files * ignore agent/uiserver folder for now * copyright headers for agent folder * Add copyright headers for acl, api and bench folders * Github Actions Migration - move go-tests workflows to GHA (#16761) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * making on pull_request * Update .github/scripts/rerun_fails_report.sh Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * make runs-on required * removing go-version param that is not used. * removing go-version param that is not used. * Modify build-distros to use medium runners (#16773) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * trying mediums * adding in script * fixing runs-on to be parameter * fixing merge conflict * changing to on push * removing whitespace * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * changing back to on pull_request --------- Co-authored-by: Dan Bond * Github Actions Migration - move verify-ci workflows to GHA (#16777) * add verify-ci workflow * adding comment and changing to on pull request. * changing to pull_requests * changing to pull_request * Apply suggestions from code review Co-authored-by: Dan Bond * [NET-3029] Migrate frontend to GHA (#16731) * changing set up to a small * using consuls own custom runner pool. --------- Co-authored-by: Dan Bond * Copyright headers for missing files/folders (#16708) * copyright headers for agent folder * fix: export ReadWriteRatesConfig struct as it needs to referenced from consul-k8s (#16766) * docs: Updates to support HCP Consul cluster peering release (#16774) * New HCP Consul documentation section + links * Establish cluster peering usage cross-link * unrelated fix to backport to v1.15 * nav correction + fixes * Tech specs fixes * specifications for headers * Tech specs fixes + alignments * sprawl edits * Tip -> note * port ENT ingress gateway upgrade tests [NET-2294] [NET-2296] (#16804) * [COMPLIANCE] Add Copyright and License Headers (#16807) * [COMPLIANCE] Add Copyright and License Headers * fix headers for generated files * ignore dist folder --------- Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> Co-authored-by: Ronald Ekambi Co-authored-by: Ronald * add order by locality failover to Consul enterprise (#16791) * ci: changes resulting from running on consul-enterprise (#16816) * changes resulting from running on consul-enterprise * removing comment line * port ENT upgrade tests flattening (#16824) * docs: raise awareness of GH-16779 (#16823) * updating command to reflect the additional package exclusions in CircleCI (#16829) * storage: fix resource leak in Watch (#16817) * Remove UI brand-loader copyright headers as they do not render appropriately (#16835) * Add sameness-group to exported-services config entries (#16836) This PR adds the sameness-group field to exported-service config entries, which allows for services to be exported to multiple destination partitions / peers easily. * Add default resolvers to disco chains based on the default sameness group (#16837) * [NET-3029] Migrate dev-* jobs to GHA (#16792) * ci: add build-artifacts workflow Signed-off-by: Dan Bond * makefile for gha dev-docker Signed-off-by: Dan Bond * use docker actions instead of make Signed-off-by: Dan Bond * Add context Signed-off-by: Dan Bond * testing push Signed-off-by: Dan Bond * set short sha Signed-off-by: Dan Bond * upload to s3 Signed-off-by: Dan Bond * rm s3 upload Signed-off-by: Dan Bond * use runner setup job Signed-off-by: Dan Bond * on push Signed-off-by: Dan Bond * testing Signed-off-by: Dan Bond * on pr Signed-off-by: Dan Bond * revert testing Signed-off-by: Dan Bond * OSS/ENT logic Signed-off-by: Dan Bond * add comments Signed-off-by: Dan Bond * Update .github/workflows/build-artifacts.yml Co-authored-by: John Murret --------- Signed-off-by: Dan Bond Co-authored-by: John Murret * add region field (#16825) * add region field * fix syntax error in test file * go fmt * go fmt * remove test * Connect CA Primary Provider refactor (#16749) * Rename Intermediate cert references to LeafSigningCert Within the Consul CA subsystem, the term "Intermediate" is confusing because the meaning changes depending on provider and datacenter (primary vs secondary). For example, when using the Consul CA the "ActiveIntermediate" may return the root certificate in a primary datacenter. At a high level, we are interested in knowing which CA is responsible for signing leaf certs, regardless of its position in a certificate chain. This rename makes the intent clearer. * Move provider state check earlier * Remove calls to GenerateLeafSigningCert GenerateLeafSigningCert (formerly known as GenerateIntermediate) is vestigial in non-Vault providers, as it simply returns the root certificate in primary datacenters. By folding Vault's intermediate cert logic into `GenerateRoot` we can encapsulate the intermediate cert handling within `newCARoot`. * Move GenerateLeafSigningCert out of PrimaryProvidder Now that the Vault Provider calls GenerateLeafSigningCert within GenerateRoot, we can remove the method from all other providers that never used it in a meaningful way. * Add test for IntermediatePEM * Rename GenerateRoot to GenerateCAChain "Root" was being overloaded in the Consul CA context, as different providers and configs resulted in a single root certificate or a chain originating from an external trusted CA. Since the Vault provider also generates intermediates, it seems more accurate to call this a CAChain. * Update changelog with patch releases (#16856) * Update changelog with patch releases * Backport missed 1.0.4 patch release to changelog * Fix typo on cli-flags.mdx (#16843) Change "segements" to segments * Allow dialer to re-establish terminated peering (#16776) Currently, if an acceptor peer deletes a peering the dialer's peering will eventually get to a "terminated" state. If the two clusters need to be re-peered the acceptor will re-generate the token but the dialer will encounter this error on the call to establish: "failed to get addresses to dial peer: failed to refresh peer server addresses, will continue to use initial addresses: there is no active peering for "<<>>"" This is because in `exchangeSecret().GetDialAddresses()` we will get an error if fetching addresses for an inactive peering. The peering shows up as inactive at this point because of the existing terminated state. Rather than checking whether a peering is active we can instead check whether it was deleted. This way users do not need to delete terminated peerings in the dialing cluster before re-establishing them. * CA mesh CA expiration to it's own section This is part of an effort to raise awareness that you need to monitor your mesh CA if coming from an external source as you'll need to manage the rotation. * Fix broken doc in consul-k8s upgrade (#16852) Signed-off-by: dttung2905 Co-authored-by: David Yu * docs: add envoy to the proxycfg diagram (#16834) * docs: add envoy to the proxycfg diagram * ci: increase deep-copy and lint-enum jobs to use large runner as they hang in ENT (#16866) * docs: add envoy to the proxycfg diagram (#16834) * docs: add envoy to the proxycfg diagram * increase dee-copy job to use large runner. disable lint-enums on ENT * set lint-enums to a large * remove redunant installation of deep-copy --------- Co-authored-by: cskh * Raft storage backend (#16619) * ad arm64 testing (#16876) * Omit false positives from 404 checker (#16881) * Remove false positives from 404 checker * fix remaining 404s * ci: fixes missing deps in frontend gha workflows (#16872) Signed-off-by: Dan Bond * always test oss and conditionally test enterprise (#16827) * temporarily disable macos-arm64 tests job in go-tests (#16898) * Resource `Write` endpoint (#16786) * Resource `Delete` endpoint (#16756) * Wasm Envoy HTTP extension (#16877) * Fix API GW broken link (#16885) * Fix API GW broken link * Update website/content/docs/api-gateway/upgrades.mdx Co-authored-by: Tu Nguyen --------- Co-authored-by: Tu Nguyen * ci: Add success jobs. make go-test-enterprise conditional. build-distros and go-tests trigger on push to main and release branches (#16905) * Add go-tests-success job and make go-test-enterprise conditional * fixing lint-32bit reference * fixing reference to -go-test-troubleshoot * add all jobs that fan out. * fixing success job to need set up * add echo to success job * adding success jobs to build-artifacts, build-distros, and frontend. * changing the name of the job in verify ci to be consistent with other workflows * enable go-tests, build-distros, and verify-ci to run on merge to main and release branches because they currently do not with just the pull_request trigger * docs: improve upgrade path guidance * fixup * backport of commit 8b549ffb7571eb8252e27105414f282b4e419b86 * backport of commit 88e60ebe3cc898b2c0817e4872eabfdbf87105b9 --------- Signed-off-by: dependabot[bot] Signed-off-by: Dan Bond Signed-off-by: dttung2905 Co-authored-by: David Yu Co-authored-by: Andrew Stucki Co-authored-by: Dan Stough Co-authored-by: Matt Keeler Co-authored-by: malizz Co-authored-by: Nitya Dhanushkodi Co-authored-by: cskh Co-authored-by: wangxinyi7 <121973291+wangxinyi7@users.noreply.github.com> Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> Co-authored-by: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Co-authored-by: Nathan Coleman Co-authored-by: Anita Akaeze Co-authored-by: Dhia Ayachi Co-authored-by: Ranjandas Co-authored-by: Kyle Havlovitz Co-authored-by: Curt Bushko Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Krastin Krastev Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> Co-authored-by: Valeriia Ruban Co-authored-by: Paul Banks Co-authored-by: Eric Haberkorn Co-authored-by: Tu Nguyen Co-authored-by: Tu Nguyen Co-authored-by: skpratt Co-authored-by: Poonam Jadhav Co-authored-by: Chris S. Kim Co-authored-by: Thomas Eckert Co-authored-by: Melisa Griffin Co-authored-by: Melisa Griffin Co-authored-by: danielehc <40759828+danielehc@users.noreply.github.com> Co-authored-by: Semir Patel Co-authored-by: claire labry Co-authored-by: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Co-authored-by: trujillo-adam Co-authored-by: Ella Cai Co-authored-by: wenincode Co-authored-by: Bryce Kalow Co-authored-by: amitchahalgits <109494649+amitchahalgits@users.noreply.github.com> Co-authored-by: Dan Upton Co-authored-by: R.B. Boyer <4903+rboyer@users.noreply.github.com> Co-authored-by: Mike Morris Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> Co-authored-by: John Eikenberry Co-authored-by: Ronald Co-authored-by: Michael Hofer Co-authored-by: Michael Wilkerson <62034708+wilkermichael@users.noreply.github.com> Co-authored-by: John Maguire Co-authored-by: Ashlee M Boyer <43934258+ashleemboyer@users.noreply.github.com> Co-authored-by: Paul Glass Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> Co-authored-by: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Co-authored-by: Ashvitha Co-authored-by: Bastien Dronneau Co-authored-by: Freddy Co-authored-by: Vipin John Wilson <37441623+vjwilson1987@users.noreply.github.com> Co-authored-by: Rosemary Wang <915624+joatmon08@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: John Murret Co-authored-by: Dan Bond Co-authored-by: brian shore Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> Co-authored-by: Ronald Ekambi Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> Co-authored-by: Michael Zalimeni Co-authored-by: Hariram Sankaran <56744845+ramramhariram@users.noreply.github.com> Co-authored-by: Dao Thanh Tung Co-authored-by: Chris Thain <32781396+cthain@users.noreply.github.com> Co-authored-by: Jared Kirschner --- .../docs/upgrading/instructions/index.mdx | 82 +++++++++---------- .../docs/upgrading/upgrade-specific.mdx | 8 ++ 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/website/content/docs/upgrading/instructions/index.mdx b/website/content/docs/upgrading/instructions/index.mdx index 8c2b3c080500..1ea7effb7353 100644 --- a/website/content/docs/upgrading/instructions/index.mdx +++ b/website/content/docs/upgrading/instructions/index.mdx @@ -10,45 +10,43 @@ description: >- This document is intended to help users who find themselves many versions behind to upgrade safely. -## Upgrade Path - -Our recommended upgrade path is to move through the following sequence of versions: - -- 0.8.5 (final 0.8.x) -- 1.2.4 (final 1.2.x) -- 1.6.10 (final 1.6.x) -- 1.8.19 (final 1.8.x) -- 1.10.12 (final 1.10.x) -- Latest 1.12.x -- Latest 1.13.x ([at least 1.13.1](/consul/docs/upgrading/upgrade-specific#service-mesh-compatibility)) -- Latest 1.14.x - -## Getting Started - -To get instructions for your upgrade, follow the instructions given below for -your _currently installed_ release series until you are on the latest current version. -The upgrade guides will mention notable changes and link to relevant changelogs – -we recommend reviewing the changelog for versions between the one you are on and the -one you are upgrading to at each step to familiarize yourself with changes. - -Select your _currently installed_ release series: -- 1.13.x: work upwards from [1.14 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-14-x) -- 1.12.x: work upwards from [1.13 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-13-x) -- 1.11.x: work upwards from [1.12 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-12-0) -- 1.10.x: work upwards from [1.11 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-11-0) -- [1.9.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) -- [1.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) -- [1.7.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) -- [1.6.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) -- [1.5.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.4.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.3.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.2.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.1.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [1.0.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [0.9.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [0.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) - -If you are using <= 0.7.x, please contact support for assistance: -- OSS users without paid support plans can request help in our [Community Forum](https://discuss.hashicorp.com/c/consul/29) -- Enterprise and OSS users with paid support plans can contact [HashiCorp Support](https://support.hashicorp.com/) +## General Upgrade Path + +Each upgrade should jump at most 2 major versions, except where +[dedicated instructions](#dedicated-instructions-for-specific-upgrade-paths) +are provided for a larger jump between specific versions. +If your upgrade path has no applicable [dedicated instructions](#dedicated-instructions-for-specific-upgrade-paths), +review the [version-specific upgrade details](/consul/docs/upgrading/upgrade-specific) +to plan your upgrade, starting from the next version and working +upwards to your target version. + +For example, to upgrade from Consul 1.12 to Consul 1.15: + +1. Upgrade to Consul 1.14 as an intermediate step. + To plan, review the upgrade details for + [1.13](/consul/docs/upgrading/upgrade-specific#consul-1-13-x) and + [1.14](/consul/docs/upgrading/upgrade-specific#consul-1-14-x). +1. Upgrade to Consul 1.15. + To plan, review the upgrade details for + [1.15](/consul/docs/upgrading/upgrade-specific#consul-1-15-x). + +## Dedicated Instructions for Specific Upgrade Paths + +The following table provides links to dedicated instructions +for directly upgrading from a version in the starting range +to a destination version. + +| Starting Version Range | Destination Version | Upgrade Instructions | +| ---------------------- | ------------------- | -------------------- | +| 1.8.0 - 1.9.17 | 1.10.12 | Refer to [upgrading to latest 1.10.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) | +| 1.6.9 - 1.8.18 | 1.8.19 | Refer to [upgrading to latest 1.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) | +| 1.2.4 - 1.6.9 | 1.6.10 | Refer to [upgrading to latest 1.6.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) | +| 0.8.5 - 1.2.3 | 1.2.4 | Refer to [upgrading to latest 1.2.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) | + +For example, to upgrade from Consul 1.3.1 to latest 1.12: +1. Upgrade to Consul 1.6.10 using the dedicated instructions. +1. Upgrade to Consul 1.8.19 using the dedicated instructions. +1. Upgrade to Consul 1.10.12 using the dedicated instructions. +1. Upgrade to latest Consul 1.12.x after consulting the + [version-specific upgrade details](/consul/docs/upgrading/upgrade-specific) + for 1.11 and 1.12. \ No newline at end of file diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 936a4cec491b..1c9a73849107 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -515,6 +515,14 @@ to Consul 1.11.11 or later to avoid the breaking nature of that change. ### Licensing Changes +You can only upgrade to Consul Enterprise 1.10 from the following Enterprise versions: +- 1.8 release series: 1.8.13+ +- 1.9 release series: 1.9.7+ + +Other versions of Consul Enterprise are not forward compatible with v1.10 and will +cause issues during the upgrade that could result in agents failing to start due to +[changes in the way we manage licenses](/consul/docs/enterprise/license/faq). + Consul Enterprise 1.10 has removed temporary licensing capabilities from the binaries found on https://releases.hashicorp.com. Servers will no longer load a license previously set through the CLI or API. Instead the license must be present in the server's configuration From 43ee53a54c453c318b8a594972f90d778a981031 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 7 Apr 2023 21:12:41 -0400 Subject: [PATCH 152/421] backport of commit 9862a83324ff8e650b973cbcb19ed55630ebe182 (#16930) Co-authored-by: cskh --- sdk/testutil/server.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/testutil/server.go b/sdk/testutil/server.go index de42a8e41afd..362c042ec83c 100644 --- a/sdk/testutil/server.go +++ b/sdk/testutil/server.go @@ -347,7 +347,13 @@ func NewTestServerConfigT(t TestingTB, cb ServerConfigCallback) (*TestServer, er // Stop stops the test Consul server, and removes the Consul data // directory once we are done. func (s *TestServer) Stop() error { - defer os.RemoveAll(s.tmpdir) + defer func() { + if noCleanup { + fmt.Println("skipping cleanup because TEST_NOCLEANUP was enabled") + } else { + os.RemoveAll(s.tmpdir) + } + }() // There was no process if s.cmd == nil { From 1f8e0083d2ffc45d18386f63dcdb9c748885fc01 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 10 Apr 2023 13:15:09 -0400 Subject: [PATCH 153/421] backport of commit 03b47d00d53d5e58dbdad4b4d67a7196b5775169 (#16858) Co-authored-by: Hariram Sankaran <56744845+ramramhariram@users.noreply.github.com> --- website/content/docs/agent/config/cli-flags.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/agent/config/cli-flags.mdx b/website/content/docs/agent/config/cli-flags.mdx index bff79cc63520..863b05f2c68b 100644 --- a/website/content/docs/agent/config/cli-flags.mdx +++ b/website/content/docs/agent/config/cli-flags.mdx @@ -323,7 +323,7 @@ information. If Consul is running on a non-default Serf LAN port, you must specify the port number in the address when using the `-retry-join` flag. Alternatively, you can specify the custom port number as the default in the agent's [`ports.serf_lan`](/consul/docs/agent/config/config-files#serf_lan_port) configuration or with the [`-serf-lan-port`](#_serf_lan_port) command line flag when starting the agent. - If your network contains network segments, refer to the [network segements documentation](/consul/docs/enterprise/network-segments/create-network-segment) for additional information. + If your network contains network segments, refer to the [network segments documentation](/consul/docs/enterprise/network-segments/create-network-segment) for additional information. Here are some examples of using `-retry-join`: From f91e170b6e555ae3b16b1c5a31a511eb1d38cd95 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 10 Apr 2023 13:51:13 -0400 Subject: [PATCH 154/421] backport of commit 4d52e5a21b2d866646a8146f8f630fb1dd6f473c (#16939) Co-authored-by: John Murret --- .github/workflows/build-distros.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index c8036a75055e..a423030be5bf 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -13,6 +13,9 @@ on: permissions: contents: read +env: + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + jobs: setup: name: Setup @@ -59,7 +62,7 @@ jobs: - name: Build run: | for os in $XC_OS; do - GOOS="$os" GOARCH=386 CGO_ENABLED=0 go build + GOOS="$os" GOARCH=386 CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" done build-amd64: @@ -83,14 +86,14 @@ jobs: - name: Build run: | for os in $XC_OS; do - GOOS="$os" GOARCH=amd64 CGO_ENABLED=0 go build + GOOS="$os" GOARCH=amd64 CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" done build-arm: needs: - setup - check-go-mod - runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} env: CGO_ENABLED: 1 GOOS: linux @@ -109,9 +112,9 @@ jobs: - run: | sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu - - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build - - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build - - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build + - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build -tags "${{ env.GOTAGS }}" + - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build -tags "${{ env.GOTAGS }}" + - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build -tags "${{ env.GOTAGS }}" # This is job is required for branch protection as a required gihub check # because GitHub actions show up as checks at the job level and not the From fa1362a3769b5744efba5ab082d57573b0106129 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 10 Apr 2023 14:56:05 -0400 Subject: [PATCH 155/421] backport of commit e9a99b170ab8a8b53dcc21b58a7ff90a1cbc5f46 (#16944) Co-authored-by: John Murret --- .circleci/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20bcf317403e..a1dd95b70a74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -997,12 +997,6 @@ jobs: - run: "echo ok" workflows: - version: 2 - # verify-ci is a no-op workflow that must run on every PR. It is used in a - # branch protection rule to detect when CI workflows are not running. - verify-ci: - jobs: [noop] - go-tests: jobs: - check-go-mod: &filter-ignore-non-go-branches From e2c17760a57a0a7b65ce93a0f83576e4c107a908 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 10 Apr 2023 17:18:08 -0400 Subject: [PATCH 156/421] Backport of ci: remove go-tests workflow from CircleCI into release/1.15.x (#16948) * no-op commit due to failed cherry-picking * ci: remove go-tests workflow from CircleCI (#16855) * remove go-tests workflow from CircleCI * add yaml anchor back --------- Co-authored-by: temp Co-authored-by: John Murret --- .circleci/config.yml | 331 +------------------------------------------ 1 file changed, 1 insertion(+), 330 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1dd95b70a74..432fd39f57f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,76 +169,6 @@ commands: - run: *notify-slack-failure jobs: - # lint consul tests - lint-consul-retry: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry - - run: *notify-slack-failure - - lint-enums: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... - - run: *notify-slack-failure - - lint-container-test-deps: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: make lint-container-test-deps - - run: *notify-slack-failure - - lint: - description: "Run golangci-lint" - parameters: - go-arch: - type: string - default: "" - docker: - - image: *GOLANG_IMAGE - resource_class: xlarge - environment: - GOTAGS: "" # No tags for OSS but there are for enterprise - GOARCH: "<>" - steps: - - checkout - - run: go env - - run: - name: Install golangci-lint - command: make lint-tools - - run: go mod download - - run: - name: lint - command: &lintcmd | - golangci-lint run --build-tags="$GOTAGS" -v - - run: - name: lint api - working_directory: api - command: *lintcmd - - run: - name: lint sdk - working_directory: sdk - command: *lintcmd - - run: - name: lint envoyextensions - working_directory: envoyextensions - command: *lintcmd - - run: - name: lint troubleshoot - working_directory: troubleshoot - command: *lintcmd - - run: - name: lint container tests - working_directory: test/integration/consul-container - command: *lintcmd - - run: *notify-slack-failure - check-go-mod: docker: - image: *GOLANG_IMAGE @@ -255,213 +185,6 @@ jobs: fi - run: *notify-slack-failure - check-generated-protobuf: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - # tput complains if this isn't set to something. - TERM: ansi - steps: - - checkout - - run: - name: Install protobuf - command: make proto-tools - - run: - name: "Protobuf Format" - command: make proto-format - - run: - command: make --always-make proto - - run: | - if ! git diff --exit-code; then - echo "Generated code was not updated correctly" - exit 1 - fi - - run: - name: "Protobuf Lint" - command: make proto-lint - - check-generated-deep-copy: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - # tput complains if this isn't set to something. - TERM: ansi - steps: - - checkout - - run: - name: Install deep-copy - command: make codegen-tools - - run: - command: make --always-make deep-copy - - run: | - if ! git diff --exit-code; then - echo "Generated code was not updated correctly" - exit 1 - fi - - go-test-arm64: - machine: - image: *UBUNTU_CI_IMAGE - resource_class: arm.large - parallelism: 4 - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - steps: - - checkout - - run: - command: | - sudo rm -rf /usr/local/go - wget https://dl.google.com/go/go${GO_VERSION}.linux-arm64.tar.gz - sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-arm64.tar.gz - - run: *install-gotestsum - - run: go mod download - - run: - name: make dev - command: | - if [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then - make dev - mkdir -p /home/circleci/bin - cp ./bin/consul /home/circleci/bin/consul - fi - - run-go-test-full: - go_test_flags: 'if ! [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' - - go-test: - docker: - - image: *GOLANG_IMAGE - resource_class: large - parallelism: 4 - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - steps: - - checkout - - run-go-test-full - - go-test-race: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - # The medium resource class (default) boxes are 2 vCPUs, 4GB RAM - # https://circleci.com/docs/2.0/configuration-reference/#docker-executor - # but we can run a little over that limit. - steps: - - checkout - - run: go mod download - - run: - name: go test -race - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - pkgs="$(go list ./... | \ - grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | \ - grep -E -v '^github.com/hashicorp/consul/command/')" - gotestsum \ - --jsonfile /tmp/jsonfile/go-test-race.log \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - -race -gcflags=all=-d=checkptr=0 \ - $pkgs - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: *notify-slack-failure - - # go-test-32bit is to catch problems where 64-bit ints must be 64-bit aligned - # to use them with sync/atomic. See https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - # Running tests with GOARCH=386 seems to be the best way to detect this - # problem. Only runs tests that are -short to limit the time we spend checking - # for these bugs. - go-test-32bit: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - steps: - - checkout - - run: go mod download - - run: - name: go test 32-bit - environment: - GOARCH: 386 - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - go env - PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./...) - gotestsum \ - --jsonfile /tmp/jsonfile/go-test-32bit.log \ - --rerun-fails=3 \ - --rerun-fails-max-failures=40 \ - --rerun-fails-report=/tmp/gotestsum-rerun-fails \ - --packages="$PACKAGE_NAMES" \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - -short - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - go-test-lib: - description: "test a library against a specific Go version" - parameters: - go-version: - type: string - path: - type: string - docker: - - image: "docker.mirror.hashicorp.services/cimg/go:<>" - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - steps: - - checkout - - attach_workspace: - at: /home/circleci/go/bin - - run: - working_directory: <> - command: go mod download - - run: - working_directory: <> - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - gotestsum \ - --format=short-verbose \ - --jsonfile /tmp/jsonfile/go-test-<>.log \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -cover -coverprofile=coverage.txt \ - ./... - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: *notify-slack-failure - # build is a templated job for build-x build-distros: &build-distros docker: @@ -997,7 +720,7 @@ jobs: - run: "echo ok" workflows: - go-tests: + build-distros: jobs: - check-go-mod: &filter-ignore-non-go-branches filters: @@ -1010,58 +733,6 @@ workflows: - /^backport\/docs\/.*/ - /^backport\/ui\/.*/ - /^backport\/mktg-.*/ - - check-generated-protobuf: *filter-ignore-non-go-branches - - check-generated-deep-copy: *filter-ignore-non-go-branches - - lint-enums: *filter-ignore-non-go-branches - - lint-container-test-deps: *filter-ignore-non-go-branches - - lint-consul-retry: *filter-ignore-non-go-branches - - lint: *filter-ignore-non-go-branches - - lint: - name: "lint-32bit" - go-arch: "386" - <<: *filter-ignore-non-go-branches - - go-test-arm64: *filter-ignore-non-go-branches - - dev-build: *filter-ignore-non-go-branches - - go-test: - requires: [dev-build] - - go-test-lib: - name: "go-test-envoyextensions" - path: envoyextensions - go-version: "1.20" - requires: [dev-build] - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-troubleshoot" - path: troubleshoot - go-version: "1.20" - requires: [dev-build] - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-api go1.19" - path: api - go-version: "1.19" - requires: [dev-build] - - go-test-lib: - name: "go-test-api go1.20" - path: api - go-version: "1.20" - requires: [dev-build] - - go-test-lib: - name: "go-test-sdk go1.19" - path: sdk - go-version: "1.19" - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-sdk go1.20" - path: sdk - go-version: "1.20" - <<: *filter-ignore-non-go-branches - - go-test-race: *filter-ignore-non-go-branches - - go-test-32bit: *filter-ignore-non-go-branches - - noop - build-distros: - jobs: - - check-go-mod: *filter-ignore-non-go-branches - build-386: &require-check-go-mod requires: - check-go-mod From 1b301679cae925190540ef1097d29f78f65345b6 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 10 Apr 2023 19:07:47 -0400 Subject: [PATCH 157/421] Backport of ci: build-artifacts - fix platform missing in manifest error into release/1.15.x (#16953) * backport of commit 37ab9a4ecf8d2fbae92299e4d8d4cbe778b166f3 * backport of commit a5fe8fb99127d0379bdd3676a175c8f6e803a805 --------- Co-authored-by: John Murret --- .github/workflows/build-artifacts.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 57d23d52f58d..632950b01d13 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -66,6 +66,8 @@ jobs: - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 with: go-version-file: 'go.mod' + + - run: go env - name: Build dev binary run: make dev @@ -92,6 +94,12 @@ jobs: file: ./build-support/docker/Consul-Dev.dockerfile labels: COMMIT_SHA=${{ github.sha }},GITHUB_BUILD_URL=${{ env.GITHUB_BUILD_URL }} push: true + # This is required or else the image is not pullable. + # See https://github.com/docker/build-push-action/issues/820 for further + # details. + # TODO - investigate further and see if we can find a solution where we + # we don't have to know to set this. + provenance: false tags: | hashicorpdev/${{ github.event.repository.name }}:${{ env.SHORT_SHA }} hashicorpdev/${{ github.event.repository.name }}:latest From 722a8e4b6ebd130cce47d0c4e2f401801af8523b Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 11 Apr 2023 12:28:25 -0400 Subject: [PATCH 158/421] Backport of APIGW: Routes with duplicate parents should be invalid into release/1.15.x (#16962) * backport of commit 7a3d2d8669a4f424d46f12c8b2abd0712a0a5950 * backport of commit ec3bcfe2bf7f9e241322979f45bad65310c55dbe * backport of commit ec34e3f5e82b5338a00a7b3b85c47778e179cb1b --------- Co-authored-by: jm96441n --- agent/structs/config_entry_routes.go | 17 +++ agent/structs/config_entry_routes_test.go | 151 +++++++++++++++++++++- 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/agent/structs/config_entry_routes.go b/agent/structs/config_entry_routes.go index ca092c07d196..d77c9dc7735f 100644 --- a/agent/structs/config_entry_routes.go +++ b/agent/structs/config_entry_routes.go @@ -1,6 +1,7 @@ package structs import ( + "errors" "fmt" "strings" @@ -140,10 +141,17 @@ func (e *HTTPRouteConfigEntry) Validate() error { APIGateway: true, } + uniques := make(map[ResourceReference]struct{}, len(e.Parents)) + for _, parent := range e.Parents { if !validParentKinds[parent.Kind] { return fmt.Errorf("unsupported parent kind: %q, must be 'api-gateway'", parent.Kind) } + + if _, ok := uniques[parent]; ok { + return errors.New("route parents must be unique") + } + uniques[parent] = struct{}{} } if err := validateConfigEntryMeta(e.Meta); err != nil { @@ -531,10 +539,19 @@ func (e *TCPRouteConfigEntry) Validate() error { if len(e.Services) > 1 { return fmt.Errorf("tcp-route currently only supports one service") } + + uniques := make(map[ResourceReference]struct{}, len(e.Parents)) + for _, parent := range e.Parents { if !validParentKinds[parent.Kind] { return fmt.Errorf("unsupported parent kind: %q, must be 'api-gateway'", parent.Kind) } + + if _, ok := uniques[parent]; ok { + return errors.New("route parents must be unique") + } + uniques[parent] = struct{}{} + } if err := validateConfigEntryMeta(e.Meta); err != nil { diff --git a/agent/structs/config_entry_routes_test.go b/agent/structs/config_entry_routes_test.go index 37c20390a311..52e18f719520 100644 --- a/agent/structs/config_entry_routes_test.go +++ b/agent/structs/config_entry_routes_test.go @@ -3,8 +3,9 @@ package structs import ( "testing" - "github.com/hashicorp/consul/acl" "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/acl" ) func TestTCPRoute(t *testing.T) { @@ -57,6 +58,78 @@ func TestTCPRoute(t *testing.T) { }, validateErr: "unsupported parent kind", }, + "duplicate parents with no listener specified": { + entry: &TCPRouteConfigEntry{ + Kind: TCPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + }, + { + Kind: "api-gateway", + Name: "gateway", + }, + }, + }, + validateErr: "route parents must be unique", + }, + "duplicate parents with listener specified": { + entry: &TCPRouteConfigEntry{ + Kind: TCPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + }, + }, + validateErr: "route parents must be unique", + }, + "almost duplicate parents with one not specifying a listener": { + entry: &TCPRouteConfigEntry{ + Kind: TCPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + }, + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + }, + }, + check: func(t *testing.T, entry ConfigEntry) { + expectedParents := []ResourceReference{ + { + Kind: APIGateway, + Name: "gateway", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + }, + { + Kind: APIGateway, + Name: "gateway", + SectionName: "same", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + }, + } + route := entry.(*TCPRouteConfigEntry) + require.Len(t, route.Parents, 2) + require.Equal(t, expectedParents[0], route.Parents[0]) + require.Equal(t, expectedParents[1], route.Parents[1]) + }, + }, } testConfigEntryNormalizeAndValidate(t, cases) } @@ -275,7 +348,8 @@ func TestHTTPRoute(t *testing.T) { { Name: "test2", Weight: -1, - }}, + }, + }, }}, }, check: func(t *testing.T, entry ConfigEntry) { @@ -284,6 +358,79 @@ func TestHTTPRoute(t *testing.T) { require.Equal(t, 1, route.Rules[0].Services[1].Weight) }, }, + + "duplicate parents with no listener specified": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + }, + { + Kind: "api-gateway", + Name: "gateway", + }, + }, + }, + validateErr: "route parents must be unique", + }, + "duplicate parents with listener specified": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + }, + }, + validateErr: "route parents must be unique", + }, + "almost duplicate parents with one not specifying a listener": { + entry: &HTTPRouteConfigEntry{ + Kind: HTTPRoute, + Name: "route-two", + Parents: []ResourceReference{ + { + Kind: "api-gateway", + Name: "gateway", + }, + { + Kind: "api-gateway", + Name: "gateway", + SectionName: "same", + }, + }, + }, + check: func(t *testing.T, entry ConfigEntry) { + expectedParents := []ResourceReference{ + { + Kind: APIGateway, + Name: "gateway", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + }, + { + Kind: APIGateway, + Name: "gateway", + SectionName: "same", + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + }, + } + route := entry.(*HTTPRouteConfigEntry) + require.Len(t, route.Parents, 2) + require.Equal(t, expectedParents[0], route.Parents[0]) + require.Equal(t, expectedParents[1], route.Parents[1]) + }, + }, } testConfigEntryNormalizeAndValidate(t, cases) } From 60ab4d87d9b2c85fc7f3c6f153b8e7a1a3fd730b Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 11 Apr 2023 17:04:27 -0400 Subject: [PATCH 159/421] Backport of ci: remove build-distros from CircleCI into release/1.15.x (#16969) * no-op commit due to failed cherry-picking * ci: remove build-distros from CircleCI (#16941) * fixing circleci config --------- Co-authored-by: temp Co-authored-by: John Murret --- .circleci/config.yml | 246 ++++++++++++++++--------------------------- 1 file changed, 92 insertions(+), 154 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 432fd39f57f6..7bccdb1da451 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,6 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + --- version: 2.1 @@ -8,16 +11,6 @@ parameters: description: "Commit to run load tests against" references: - paths: - test-results: &TEST_RESULTS_DIR /tmp/test-results - environment: &ENVIRONMENT - TEST_RESULTS_DIR: *TEST_RESULTS_DIR - EMAIL: noreply@hashicorp.com - GIT_AUTHOR_NAME: circleci-consul - GIT_COMMITTER_NAME: circleci-consul - S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 - BASH_ENV: .circleci/bash_env.sh - GO_VERSION: 1.20.1 envoy-versions: &supported_envoy_versions - &default_envoy_version "1.22.7" - "1.23.4" @@ -32,15 +25,37 @@ references: - "1.11.6" - "1.10.9" - "1.9.10" + consul-versions: &consul_versions + - "1.15" + - "1.14" images: # When updating the Go version, remember to also update the versions in the # workflows section for go-test-lib jobs. go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.20.1 - ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers + ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:16-browsers ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02 + + paths: + test-results: &TEST_RESULTS_DIR /tmp/test-results + cache: yarn: &YARN_CACHE_KEY consul-ui-v9-{{ checksum "ui/yarn.lock" }} + consul_exec_contexts: &consul_exec_contexts + - team-consul + - consul-enterprise-licensing + + environment: &ENVIRONMENT + TEST_RESULTS_DIR: *TEST_RESULTS_DIR + EMAIL: noreply@hashicorp.com + GIT_AUTHOR_NAME: circleci-consul + GIT_COMMITTER_NAME: circleci-consul + S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 + S3_ARTIFACT_BUCKET_CLOUD: consul-enterprise-dev-artifacts-v2 + BASH_ENV: .circleci/bash_env.sh + GOPRIVATE: github.com/hashicorp + GO_VERSION: 1.20.1 + steps: install-gotestsum: &install-gotestsum name: install gotestsum @@ -207,58 +222,6 @@ jobs: path: ./pkg/bin - run: *notify-slack-failure - # build all 386 architecture supported OS binaries - build-386: - <<: *build-distros - environment: - <<: *build-env - XC_OS: "freebsd linux windows" - GOARCH: "386" - - # build all amd64 architecture supported OS binaries - build-amd64: - <<: *build-distros - environment: - <<: *build-env - XC_OS: "darwin freebsd linux solaris windows" - GOARCH: "amd64" - - # build all arm/arm64 architecture supported OS binaries - build-arm: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - CGO_ENABLED: 1 - GOOS: linux - steps: - - checkout - - run: - command: | - sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list # workaround for https://github.com/actions/runner-images/issues/1983 - sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu - - run: - environment: - GOARM: 5 - CC: arm-linux-gnueabi-gcc - GOARCH: arm - command: go build -o ./pkg/bin/linux_armel/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - run: - environment: - GOARM: 6 - CC: arm-linux-gnueabihf-gcc - GOARCH: arm - command: go build -o ./pkg/bin/linux_armhf/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - run: - environment: - CC: aarch64-linux-gnu-gcc - GOARCH: arm64 - command: go build -o ./pkg/bin/linux_aarch64/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - store_artifacts: - path: ./pkg/bin - - run: *notify-slack-failure - # create a development build dev-build: docker: @@ -423,64 +386,6 @@ jobs: JOBS: 2 # limit parallelism for broccoli-babel-transpiler CONSUL_NSPACES_ENABLED: 1 - # rebuild UI for packaging - ember-build-prod: - docker: - - image: *EMBER_IMAGE - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - run: cd ui && make - - # saves the build to a workspace to be passed to a downstream job - - persist_to_workspace: - root: ui - paths: - - packages/consul-ui/dist - - run: *notify-slack-failure - - # commits static assets to git - publish-static-assets: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - add_ssh_keys: # needs a key to push updated static asset commit back to github - fingerprints: - - "94:03:9e:8b:24:7f:36:60:00:30:b8:32:ed:e7:59:10" - - attach_workspace: - at: . - - run: - name: move compiled ui files to agent/uiserver - command: | - rm -rf agent/uiserver/dist - mv packages/consul-ui/dist agent/uiserver - - run: - name: commit agent/uiserver/dist/ if there are UI changes - command: | - # check if there are any changes in ui/ - # if there are, we commit the ui static asset file - # HEAD^! is shorthand for HEAD^..HEAD (parent of HEAD and HEAD) - if ! git diff --quiet --exit-code HEAD^! ui/; then - git config --local user.email "github-team-consul-core@hashicorp.com" - git config --local user.name "hc-github-team-consul-core" - - # -B resets the CI branch to main which may diverge history - # but we will force push anyways. - git checkout -B ci/main-assetfs-build main - - short_sha=$(git rev-parse --short HEAD) - git add agent/uiserver/dist/ - git commit -m "auto-updated agent/uiserver/dist/ from commit ${short_sha}" - git push --force origin ci/main-assetfs-build - else - echo "no UI changes so no static assets to publish" - fi - - run: *notify-slack-failure - # run node tests node-tests: docker: @@ -625,6 +530,70 @@ jobs: path: *TEST_RESULTS_DIR - run: *notify-slack-failure + upgrade-integration-test: + machine: + image: *UBUNTU_CI_IMAGE + docker_layer_caching: true + parallelism: 3 + resource_class: large + parameters: + consul-version: + type: enum + enum: *consul_versions + environment: + CONSUL_VERSION: << parameters.consul-version >> + steps: + - checkout + # Get go binary from workspace + - attach_workspace: + at: . + # Build the consul:local image from the already built binary + - run: + command: | + sudo rm -rf /usr/local/go + wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz + environment: + <<: *ENVIRONMENT + - run: *install-gotestsum + - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . + - run: + name: Upgrade Integration Tests + command: | + mkdir -p /tmp/test-results/ + cd ./test/integration/consul-container + docker run --rm consul:local consul version + gotestsum \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${GOTAGS}" \ + -timeout=30m \ + -json \ + ./.../upgrade/ \ + --target-image consul \ + --target-version local \ + --latest-image consul \ + --latest-version $CONSUL_VERSION + ls -lrt + environment: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + - store_test_results: + path: *TEST_RESULTS_DIR + - store_artifacts: + path: *TEST_RESULTS_DIR + - run: *notify-slack-failure + envoy-integration-test: &ENVOY_TESTS machine: image: *UBUNTU_CI_IMAGE @@ -720,9 +689,9 @@ jobs: - run: "echo ok" workflows: - build-distros: + test-integrations: jobs: - - check-go-mod: &filter-ignore-non-go-branches + - dev-build: &filter-ignore-non-go-branches filters: branches: ignore: @@ -733,37 +702,6 @@ workflows: - /^backport\/docs\/.*/ - /^backport\/ui\/.*/ - /^backport\/mktg-.*/ - - build-386: &require-check-go-mod - requires: - - check-go-mod - - build-amd64: *require-check-go-mod - - build-arm: *require-check-go-mod - # every commit on main will have a rebuilt UI - - frontend-cache: - filters: - branches: - only: - - main - - ember-build-prod: - requires: - - frontend-cache - - publish-static-assets: - requires: - - ember-build-prod - - dev-build: - requires: - - ember-build-prod - - dev-upload-s3: - requires: - - dev-build - - dev-upload-docker: - requires: - - dev-build - context: consul-ci - - noop - test-integrations: - jobs: - - dev-build: *filter-ignore-non-go-branches - dev-upload-s3: &dev-upload requires: - dev-build From b787a829a3fe99d85c8bf1e9798d875050d88b6f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 12 Apr 2023 01:12:03 -0400 Subject: [PATCH 160/421] Backport of ci: split frontend ember jobs into release/1.15.x (#16976) * backport of commit 331baedf6f39b8c89c3343ab3e6faec23c5894cd * backport of commit 648d178e5eaf1efd662eeb945e676cbb9183f4f6 --------- Co-authored-by: Dan Bond --- .github/workflows/frontend.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index b6451a378c36..2758e57c4aba 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -75,10 +75,14 @@ jobs: ember-build-test: needs: setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + strategy: + matrix: + partition: [1, 2, 3, 4] env: EMBER_TEST_REPORT: test-results/report-oss.xml # outputs test report for CircleCI test summary EMBER_TEST_PARALLEL: true # enables test parallelization with ember-exam CONSUL_NSPACES_ENABLED: ${{ endsWith(github.repository, '-enterprise') && 1 || 0 }} # NOTE: this should be 1 in ENT. + JOBS: 2 # limit parallelism for broccoli-babel-transpiler steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -92,17 +96,20 @@ jobs: - name: Install Chrome uses: browser-actions/setup-chrome@29abc1a83d1d71557708563b4bc962d0f983a376 # pin@v1.2.1 - # Install dependencies. - - name: install yarn packages + - name: Install dependencies working-directory: ui run: make deps - - working-directory: ui/packages/consul-ui - run: | - make build-ci - node_modules/.bin/ember exam --path dist --silent -r xunit + - name: Build CI + working-directory: ui/packages/consul-ui + run: make build-ci - - working-directory: ui/packages/consul-ui + - name: Ember exam + working-directory: ui/packages/consul-ui + run: node_modules/.bin/ember exam --split=4 --partition=${{ matrix.partition }} --path dist --silent -r xunit + + - name: Test Coverage CI + working-directory: ui/packages/consul-ui run: make test-coverage-ci # This is job is required for branch protection as a required gihub check From d18e88056d0ef84c244b2dffaac4ae59318cc10f Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 12 Apr 2023 18:08:45 -0400 Subject: [PATCH 161/421] Backport of Update list of Envoy versions into release/1.15.x (#16989) * backport of commit a59f68f51e67f333e4af6f716e102e074a1a5ef2 * backport of commit dace20a9e4074c2110d99f448ad89b952a64a43b * backport of commit ad6e2a472b9a1258a3adccdd471e9468dd19daf1 * backport of commit b48c42a6cc0e002af4586b3cbe25a1fb0e8b51a0 * backport of commit b913f60dee80743754d91164a8921eb3a4670d8a --------- Co-authored-by: Nathan Coleman --- .changelog/16889.txt | 3 +++ .circleci/config.yml | 8 ++++---- envoyextensions/xdscommon/envoy_versioning_test.go | 8 ++++---- envoyextensions/xdscommon/proxysupport.go | 8 ++++---- website/content/docs/connect/proxies/envoy.mdx | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 .changelog/16889.txt diff --git a/.changelog/16889.txt b/.changelog/16889.txt new file mode 100644 index 000000000000..67a859a430ae --- /dev/null +++ b/.changelog/16889.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: update supported envoy versions to 1.22.11, 1.23.8, 1.24.6, 1.25.4 +``` diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bccdb1da451..d1218e55899d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,10 +12,10 @@ parameters: references: envoy-versions: &supported_envoy_versions - - &default_envoy_version "1.22.7" - - "1.23.4" - - "1.24.2" - - "1.25.1" + - &default_envoy_version "1.22.11" + - "1.23.8" + - "1.24.6" + - "1.25.4" nomad-versions: &supported_nomad_versions - &default_nomad_version "1.3.3" - "1.2.10" diff --git a/envoyextensions/xdscommon/envoy_versioning_test.go b/envoyextensions/xdscommon/envoy_versioning_test.go index e20a2ca8ceef..600dd2f56cc4 100644 --- a/envoyextensions/xdscommon/envoy_versioning_test.go +++ b/envoyextensions/xdscommon/envoy_versioning_test.go @@ -136,10 +136,10 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) { } */ for _, v := range []string{ - "1.22.0", "1.22.1", "1.22.2", "1.22.3", "1.22.4", "1.22.5", - "1.23.0", "1.23.1", "1.23.2", "1.23.3", "1.23.4", - "1.24.0", "1.24.1", "1.24.2", - "1.25.0", "1.25.1", + "1.22.0", "1.22.1", "1.22.2", "1.22.3", "1.22.4", "1.22.5", "1.22.6", "1.22.7", "1.22.8", "1.22.9", "1.22.10", "1.22.11", + "1.23.0", "1.23.1", "1.23.2", "1.23.3", "1.23.4", "1.23.5", "1.23.6", "1.23.7", "1.23.8", + "1.24.0", "1.24.1", "1.24.2", "1.24.3", "1.24.4", "1.24.5", "1.24.6", + "1.25.0", "1.25.1", "1.25.2", "1.25.3", "1.25.4", } { cases[v] = testcase{expect: SupportedProxyFeatures{}} } diff --git a/envoyextensions/xdscommon/proxysupport.go b/envoyextensions/xdscommon/proxysupport.go index bedc0608bfd3..90d6b378b5da 100644 --- a/envoyextensions/xdscommon/proxysupport.go +++ b/envoyextensions/xdscommon/proxysupport.go @@ -9,10 +9,10 @@ import "strings" // // see: https://www.consul.io/docs/connect/proxies/envoy#supported-versions var EnvoyVersions = []string{ - "1.25.1", - "1.24.2", - "1.23.4", - "1.22.5", + "1.25.4", + "1.24.6", + "1.23.8", + "1.22.11", } // UnsupportedEnvoyVersions lists any unsupported Envoy versions (mainly minor versions) that fall diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 73e7e3c535c0..15b7856a0826 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -39,7 +39,7 @@ Consul supports **four major Envoy releases** at the beginning of each major Con | Consul Version | Compatible Envoy Versions | | ------------------- | -----------------------------------------------------------------------------------| -| 1.15.x | 1.25.1, 1.24.2, 1.23.4, 1.22.5 | +| 1.15.x | 1.25.4, 1.24.6, 1.23.8, 1.22.11 | | 1.14.x | 1.24.0, 1.23.1, 1.22.5, 1.21.5 | | 1.13.x | 1.23.1, 1.22.5, 1.21.5, 1.20.7 | From 816f8151abfa8d98ee9e54fd8151dd1b8fbc6ea9 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 12 Apr 2023 18:23:33 -0400 Subject: [PATCH 162/421] backport of commit 91be1c3e35880e856a7d19f5cba13c9736a836c1 (#16990) Co-authored-by: cskh --- test/integration/consul-container/go.mod | 1 + test/integration/consul-container/go.sum | 3 +++ .../consul-container/libs/cluster/agent.go | 13 ++++++++++++ .../consul-container/libs/cluster/builder.go | 5 +++++ .../libs/cluster/container.go | 21 +++++++++++++++---- 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 35174f5fd0b4..e13d6e4f1fd3 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -15,6 +15,7 @@ require ( github.com/hashicorp/serf v0.10.1 github.com/itchyny/gojq v0.12.9 github.com/mitchellh/copystructure v1.2.0 + github.com/otiai10/copy v1.10.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum index e554f8ad3f70..010695dea029 100644 --- a/test/integration/consul-container/go.sum +++ b/test/integration/consul-container/go.sum @@ -630,6 +630,9 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/otiai10/copy v1.10.0 h1:znyI7l134wNg/wDktoVQPxPkgvhDfGCYUasey+h0rDQ= +github.com/otiai10/copy v1.10.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 8dfa496d8bff..ec525780bb1d 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -37,6 +37,19 @@ type Agent interface { // // Constructed by (Builder).ToAgentConfig() type Config struct { + // NodeName is set for the consul agent name and container name + // Equivalent to the -node command-line flag. + // If empty, a randam name will be generated + NodeName string + // NodeID is used to configure node_id in agent config file + // Equivalent to the -node-id command-line flag. + // If empty, a randam name will be generated + NodeID string + + // ExternalDataDir is data directory to copy consul data from, if set. + // This directory contains subdirectories like raft, serf, services + ExternalDataDir string + ScratchDir string CertVolume string CACert string diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index 2807985e17e5..9cec08560da1 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -269,6 +269,11 @@ func (b *Builder) Peering(enable bool) *Builder { return b } +func (b *Builder) NodeID(nodeID string) *Builder { + b.conf.Set("node_id", nodeID) + return b +} + func (b *Builder) Partition(name string) *Builder { b.conf.Set("partition", name) return b diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 4e3e1c39901c..dfda21615978 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -13,6 +13,7 @@ import ( goretry "github.com/avast/retry-go" dockercontainer "github.com/docker/docker/api/types/container" "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" "github.com/pkg/errors" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -91,11 +92,15 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po return nil, err } - consulType := "client" - if pc.Server { - consulType = "server" + name := config.NodeName + if name == "" { + // Generate a random name for the agent + consulType := "client" + if pc.Server { + consulType = "server" + } + name = utils.RandName(fmt.Sprintf("%s-consul-%s-%d", pc.Datacenter, consulType, index)) } - name := utils.RandName(fmt.Sprintf("%s-consul-%s-%d", pc.Datacenter, consulType, index)) // Inject new Agent name config.Cmd = append(config.Cmd, "-node", name) @@ -108,6 +113,14 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po return nil, fmt.Errorf("error chowning data directory %s: %w", tmpDirData, err) } + if config.ExternalDataDir != "" { + // copy consul persistent state from an external dir + err := copy.Copy(config.ExternalDataDir, tmpDirData) + if err != nil { + return nil, fmt.Errorf("error copying persistent data from %s: %w", config.ExternalDataDir, err) + } + } + var caCertFileForAPI string if config.CACert != "" { caCertFileForAPI = filepath.Join(config.ScratchDir, "ca.pem") From 79ed240e54e99b4b762b616518d8366b8b4485bd Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 12 Apr 2023 18:40:49 -0400 Subject: [PATCH 163/421] circleci: remove frontend (#16988) Signed-off-by: Dan Bond Co-authored-by: Dan Bond --- .circleci/config.yml | 208 ++++--------------------------------------- 1 file changed, 18 insertions(+), 190 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1218e55899d..33d3b45f3623 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,16 @@ parameters: description: "Commit to run load tests against" references: + paths: + test-results: &TEST_RESULTS_DIR /tmp/test-results + environment: &ENVIRONMENT + TEST_RESULTS_DIR: *TEST_RESULTS_DIR + EMAIL: noreply@hashicorp.com + GIT_AUTHOR_NAME: circleci-consul + GIT_COMMITTER_NAME: circleci-consul + S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 + BASH_ENV: .circleci/bash_env.sh + GO_VERSION: 1.20.1 envoy-versions: &supported_envoy_versions - &default_envoy_version "1.22.11" - "1.23.8" @@ -26,36 +36,17 @@ references: - "1.10.9" - "1.9.10" consul-versions: &consul_versions - - "1.15" - "1.14" + - "1.15" images: # When updating the Go version, remember to also update the versions in the # workflows section for go-test-lib jobs. go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.20.1 ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:16-browsers ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02 - - paths: - test-results: &TEST_RESULTS_DIR /tmp/test-results - cache: yarn: &YARN_CACHE_KEY consul-ui-v9-{{ checksum "ui/yarn.lock" }} - consul_exec_contexts: &consul_exec_contexts - - team-consul - - consul-enterprise-licensing - - environment: &ENVIRONMENT - TEST_RESULTS_DIR: *TEST_RESULTS_DIR - EMAIL: noreply@hashicorp.com - GIT_AUTHOR_NAME: circleci-consul - GIT_COMMITTER_NAME: circleci-consul - S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 - S3_ARTIFACT_BUCKET_CLOUD: consul-enterprise-dev-artifacts-v2 - BASH_ENV: .circleci/bash_env.sh - GOPRIVATE: github.com/hashicorp - GO_VERSION: 1.20.1 - steps: install-gotestsum: &install-gotestsum name: install gotestsum @@ -337,143 +328,6 @@ jobs: path: *TEST_RESULTS_DIR - run: *notify-slack-failure - # build frontend yarn cache - frontend-cache: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - # cache yarn deps - - restore_cache: - key: *YARN_CACHE_KEY - - - run: - name: install yarn packages - command: cd ui && make deps - - - save_cache: - key: *YARN_CACHE_KEY - paths: - - ui/node_modules - - ui/packages/consul-ui/node_modules - - run: *notify-slack-failure - - # build ember so frontend tests run faster - ember-build-oss: &ember-build-oss - docker: - - image: *EMBER_IMAGE - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - CONSUL_NSPACES_ENABLED: 0 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - run: cd ui/packages/consul-ui && make build-ci - - # saves the build to a workspace to be passed to a downstream job - - persist_to_workspace: - root: ui - paths: - - packages/consul-ui/dist - - run: *notify-slack-failure - - # build ember so frontend tests run faster - ember-build-ent: - <<: *ember-build-oss - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - CONSUL_NSPACES_ENABLED: 1 - - # run node tests - node-tests: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: make test-node - - run: *notify-slack-failure - # run yarn workspace wide checks/tests - workspace-tests: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui - command: make test-workspace - - run: *notify-slack-failure - - # run ember frontend tests - ember-test-oss: - docker: - - image: *EMBER_IMAGE - environment: - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary - EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam - CONSUL_NSPACES_ENABLED: 0 - parallelism: 4 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit - - store_test_results: - path: ui/packages/consul-ui/test-results - - run: *notify-slack-failure - - # run ember frontend tests - ember-test-ent: - docker: - - image: *EMBER_IMAGE - environment: - EMBER_TEST_REPORT: test-results/report-ent.xml #outputs test report for CircleCI test summary - EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam - CONSUL_NSPACES_ENABLED: 1 - parallelism: 4 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit - - store_test_results: - path: ui/packages/consul-ui/test-results - - run: *notify-slack-failure - - # run ember frontend unit tests to produce coverage report - ember-coverage: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: make test-coverage-ci - - run: *notify-slack-failure - compatibility-integration-test: machine: image: *UBUNTU_CI_IMAGE @@ -511,7 +365,7 @@ jobs: -p=4 \ -timeout=30m \ -json \ - ./... \ + `go list ./... | grep -v upgrade` \ --target-image consul \ --target-version local \ --latest-image consul \ @@ -734,37 +588,11 @@ workflows: - compatibility-integration-test: requires: - dev-build - - noop - frontend: - jobs: - - frontend-cache: - filters: - branches: - only: - - main - - /^ui\/.*/ - - /^backport\/ui\/.*/ - - workspace-tests: - requires: - - frontend-cache - - node-tests: - requires: - - frontend-cache - - ember-build-oss: - requires: - - frontend-cache - - ember-build-ent: + - upgrade-integration-test: requires: - - frontend-cache - - ember-test-oss: - requires: - - ember-build-oss - - ember-test-ent: - requires: - - ember-build-ent - # ember-coverage in CI uses the dist/ folder to run tests so it requires - # either/or ent/oss to be built first - - ember-coverage: - requires: - - ember-build-ent + - dev-build + matrix: + parameters: + consul-version: *consul_versions + - noop From fc1ba0f203a8844df1d5c8f6429e52ad72d5185b Mon Sep 17 00:00:00 2001 From: Poonam Jadhav Date: Thu, 13 Apr 2023 11:42:08 -0400 Subject: [PATCH 164/421] feat: add reporting config with reload (#16977) --- agent/agent.go | 8 ++ agent/agent_oss_test.go | 46 +++++++++++ agent/config/builder_oss.go | 4 + agent/config/builder_oss_test.go | 13 +++ agent/config/config.go | 11 +++ agent/config/runtime.go | 10 +++ agent/config/runtime_oss_test.go | 81 +++++++++++++++++++ .../TestRuntimeConfig_Sanitize.golden | 5 ++ agent/config/testdata/full-config.hcl | 5 ++ agent/config/testdata/full-config.json | 5 ++ agent/consul/config.go | 11 +++ agent/consul/server.go | 2 + agent/consul/server_oss.go | 4 + agent/consul/server_oss_test.go | 43 ++++++++++ 14 files changed, 248 insertions(+) create mode 100644 agent/agent_oss_test.go create mode 100644 agent/consul/server_oss_test.go diff --git a/agent/agent.go b/agent/agent.go index 9c85f065ec2b..b37dc62ffbf1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1494,7 +1494,10 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co cfg.RequestLimitsReadRate = runtimeCfg.RequestLimitsReadRate cfg.RequestLimitsWriteRate = runtimeCfg.RequestLimitsWriteRate + cfg.Reporting.License.Enabled = runtimeCfg.Reporting.License.Enabled + enterpriseConsulConfig(cfg, runtimeCfg) + return cfg, nil } @@ -4185,6 +4188,11 @@ func (a *Agent) reloadConfigInternal(newCfg *config.RuntimeConfig) error { HeartbeatTimeout: newCfg.ConsulRaftHeartbeatTimeout, ElectionTimeout: newCfg.ConsulRaftElectionTimeout, RaftTrailingLogs: newCfg.RaftTrailingLogs, + Reporting: consul.Reporting{ + License: consul.License{ + Enabled: newCfg.Reporting.License.Enabled, + }, + }, } if err := a.delegate.ReloadConfig(cc); err != nil { return err diff --git a/agent/agent_oss_test.go b/agent/agent_oss_test.go new file mode 100644 index 000000000000..ceb90beb0634 --- /dev/null +++ b/agent/agent_oss_test.go @@ -0,0 +1,46 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !consulent +// +build !consulent + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAgent_consulConfig_Reporting(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + hcl := ` + reporting { + license { + enabled = true + } + } + ` + a := NewTestAgent(t, hcl) + defer a.Shutdown() + require.Equal(t, false, a.consulConfig().Reporting.License.Enabled) +} + +func TestAgent_consulConfig_Reporting_Default(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + hcl := ` + reporting { + } + ` + a := NewTestAgent(t, hcl) + defer a.Shutdown() + require.Equal(t, false, a.consulConfig().Reporting.License.Enabled) +} diff --git a/agent/config/builder_oss.go b/agent/config/builder_oss.go index ce6e8d44ce0b..f0fcc30ae4ea 100644 --- a/agent/config/builder_oss.go +++ b/agent/config/builder_oss.go @@ -57,6 +57,10 @@ func validateEnterpriseConfigKeys(config *Config) []error { add("license_path") config.LicensePath = nil } + if config.Reporting.License.Enabled != nil { + add("reporting.license.enabled") + config.Reporting.License.Enabled = nil + } return result } diff --git a/agent/config/builder_oss_test.go b/agent/config/builder_oss_test.go index 2fd5f50ad7c3..1fdba09e6028 100644 --- a/agent/config/builder_oss_test.go +++ b/agent/config/builder_oss_test.go @@ -107,6 +107,19 @@ func TestValidateEnterpriseConfigKeys(t *testing.T) { require.Empty(t, c.LicensePath) }, }, + "reporting.license.enabled": { + config: Config{ + Reporting: Reporting{ + License: License{ + Enabled: &boolVal, + }, + }, + }, + badKeys: []string{"reporting.license.enabled"}, + check: func(t *testing.T, c *Config) { + require.Nil(t, c.Reporting.License.Enabled) + }, + }, "multi": { config: Config{ ReadReplica: &boolVal, diff --git a/agent/config/config.go b/agent/config/config.go index 6ed4e9616f0e..348dfbaddba8 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -291,6 +291,9 @@ type Config struct { LicensePollMaxTime *string `mapstructure:"license_poll_max_time" json:"-"` LicenseUpdateBaseTime *string `mapstructure:"license_update_base_time" json:"-"` LicenseUpdateMaxTime *string `mapstructure:"license_update_max_time" json:"-"` + + // license reporting + Reporting Reporting `mapstructure:"reporting" json:"-"` } type GossipLANConfig struct { @@ -943,3 +946,11 @@ type RaftBoltDBConfigRaw struct { type RaftWALConfigRaw struct { SegmentSizeMB *int `mapstructure:"segment_size_mb" json:"segment_size_mb,omitempty"` } + +type License struct { + Enabled *bool `mapstructure:"enabled"` +} + +type Reporting struct { + License License `mapstructure:"license"` +} diff --git a/agent/config/runtime.go b/agent/config/runtime.go index b0d9cf436e55..24b39507723d 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -1479,9 +1479,19 @@ type RuntimeConfig struct { // here so that tests can use a smaller value. LocalProxyConfigResyncInterval time.Duration + Reporting ReportingConfig + EnterpriseRuntimeConfig } +type LicenseConfig struct { + Enabled bool +} + +type ReportingConfig struct { + License LicenseConfig +} + type AutoConfig struct { Enabled bool IntroToken string diff --git a/agent/config/runtime_oss_test.go b/agent/config/runtime_oss_test.go index 6274511bc038..8d2e71e1c605 100644 --- a/agent/config/runtime_oss_test.go +++ b/agent/config/runtime_oss_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/hashicorp/consul/sdk/testutil" + "github.com/stretchr/testify/require" ) var testRuntimeConfigSanitizeExpectedFilename = "TestRuntimeConfig_Sanitize.golden" @@ -28,6 +29,7 @@ var enterpriseConfigKeyWarnings = []string{ enterpriseConfigKeyError{key: "acl.msp_disable_bootstrap"}.Error(), enterpriseConfigKeyError{key: "acl.tokens.managed_service_provider"}.Error(), enterpriseConfigKeyError{key: "audit"}.Error(), + enterpriseConfigKeyError{key: "reporting.license.enabled"}.Error(), } // OSS-only equivalent of TestConfigFlagsAndEdgecases @@ -84,3 +86,82 @@ func TestLoad_IntegrationWithFlags_OSS(t *testing.T) { } } } + +func TestLoad_ReportingConfig(t *testing.T) { + dir := testutil.TempDir(t, t.Name()) + + t.Run("load from JSON defaults to false", func(t *testing.T) { + content := `{ + "reporting": {} + }` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.json", + Format: "json", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 0) + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) + + t.Run("load from HCL defaults to false", func(t *testing.T) { + content := ` + reporting {} + ` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.hcl", + Format: "hcl", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 0) + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) + + t.Run("with value set returns warning and defaults to false", func(t *testing.T) { + content := `reporting { + license { + enabled = true + } + }` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.hcl", + Format: "hcl", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 1) + require.Contains(t, result.Warnings[0], "\"reporting.license.enabled\" is a Consul Enterprise configuration and will have no effect") + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) +} diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index fff09fa343f6..69d03a564f8d 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -289,6 +289,11 @@ "ReconnectTimeoutLAN": "0s", "ReconnectTimeoutWAN": "0s", "RejoinAfterLeave": false, + "Reporting": { + "License": { + "Enabled": false + } + }, "RequestLimitsMode": 0, "RequestLimitsReadRate": 0, "RequestLimitsWriteRate": 0, diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index dced4781c91e..f08c1a071747 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -372,6 +372,11 @@ reconnect_timeout = "23739s" reconnect_timeout_wan = "26694s" recursors = [ "63.38.39.58", "92.49.18.18" ] rejoin_after_leave = true +reporting = { + license = { + enabled = false + } +} retry_interval = "8067s" retry_interval_wan = "28866s" retry_join = [ "pbsSFY7U", "l0qLtWij" ] diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 7c36981561a7..f98bfe4dab2b 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -428,6 +428,11 @@ "92.49.18.18" ], "rejoin_after_leave": true, + "reporting": { + "license": { + "enabled": false + } + }, "retry_interval": "8067s", "retry_interval_wan": "28866s", "retry_join": [ diff --git a/agent/consul/config.go b/agent/consul/config.go index f114dcecfc8a..1721412f5df6 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -436,6 +436,8 @@ type Config struct { PeeringTestAllowPeerRegistrations bool + Reporting Reporting + // Embedded Consul Enterprise specific configuration *EnterpriseConfig } @@ -671,6 +673,7 @@ type ReloadableConfig struct { RaftTrailingLogs int HeartbeatTimeout time.Duration ElectionTimeout time.Duration + Reporting Reporting } type RaftLogStoreConfig struct { @@ -693,3 +696,11 @@ type RaftBoltDBConfig struct { type WALConfig struct { SegmentSize int } + +type License struct { + Enabled bool +} + +type Reporting struct { + License License +} diff --git a/agent/consul/server.go b/agent/consul/server.go index 5a68e780b415..d2c8c3cfe332 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -1746,6 +1746,8 @@ func (s *Server) ReloadConfig(config ReloadableConfig) error { return err } + s.updateReportingConfig(config) + s.rpcLimiter.Store(rate.NewLimiter(config.RPCRateLimit, config.RPCMaxBurst)) if config.RequestLimits != nil { diff --git a/agent/consul/server_oss.go b/agent/consul/server_oss.go index 4ae524b65c04..67731e450195 100644 --- a/agent/consul/server_oss.go +++ b/agent/consul/server_oss.go @@ -174,3 +174,7 @@ func addSerfMetricsLabels(conf *serf.Config, wan bool, segment string, partition conf.MetricLabels = append(conf.MetricLabels, networkMetric) } + +func (s *Server) updateReportingConfig(config ReloadableConfig) { + // no-op +} diff --git a/agent/consul/server_oss_test.go b/agent/consul/server_oss_test.go new file mode 100644 index 000000000000..d5a2aff08208 --- /dev/null +++ b/agent/consul/server_oss_test.go @@ -0,0 +1,43 @@ +//go:build !consulent +// +build !consulent + +package consul + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/testrpc" +) + +func TestAgent_ReloadConfig_Reporting(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + dir1, s := testServerWithConfig(t, func(c *Config) { + c.Reporting.License.Enabled = false + }) + defer os.RemoveAll(dir1) + defer s.Shutdown() + + testrpc.WaitForTestAgent(t, s.RPC, "dc1") + + require.Equal(t, false, s.config.Reporting.License.Enabled) + + rc := ReloadableConfig{ + Reporting: Reporting{ + License: License{ + Enabled: true, + }, + }, + } + + require.NoError(t, s.ReloadConfig(rc)) + + // Check config reload is no-op + require.Equal(t, false, s.config.Reporting.License.Enabled) +} From 50b71b934baecd12be9a3a5a5df5a27ae60eb491 Mon Sep 17 00:00:00 2001 From: cskh Date: Thu, 13 Apr 2023 13:23:57 -0400 Subject: [PATCH 165/421] upgrade test: fix missing upgrade version 1.13 (#16995) --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33d3b45f3623..3e2a1832ec05 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,6 +36,7 @@ references: - "1.10.9" - "1.9.10" consul-versions: &consul_versions + - "1.13" - "1.14" - "1.15" images: From 2b961fa4814f108f776afc9a6f27d90a6c1e1c80 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 14 Apr 2023 16:15:55 -0400 Subject: [PATCH 166/421] backport of commit d9c8e93c7686dd8dbf8e05a705ab5d4443263890 (#17008) Co-authored-by: trujillo-adam --- .../docs/troubleshoot/common-errors.mdx | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/website/content/docs/troubleshoot/common-errors.mdx b/website/content/docs/troubleshoot/common-errors.mdx index d0e3328aad04..3c5ff5b076ad 100644 --- a/website/content/docs/troubleshoot/common-errors.mdx +++ b/website/content/docs/troubleshoot/common-errors.mdx @@ -5,16 +5,16 @@ description: >- Troubleshoot issues based on the error message. Common errors result from failed actions, timeouts, multiple entries, bad and expired certificates, invalid characters, syntax parsing, malformed responses, and exceeded deadlines. --- -# Common Error Messages +# Common error messages -When installing and running Consul, there are some common messages you might see. Usually they indicate an issue in your network or in your server's configuration. Some of the more common errors and their solutions are listed below. +This topic describes common messages that may appear when installing and running Consul. Errors usually they indicate an issue in your network or in your server's configuration. Refer to the [Troubleshooting Guide][troubleshooting] for help resolving error messages that do not appear on this page. -If you are getting an error message you don't see listed on this page, please consider following our general [Troubleshooting Guide][troubleshooting]. - -For common errors messages related to Kubernetes, please go to [Common errors on Kubernetes](#common-errors-on-kubernetes). +For common errors messages related to Kubernetes, refer to [Common errors on Kubernetes](#common-errors-on-kubernetes). ## Configuration file errors +The following errors are related to misconfigured files. + ### Multiple network interfaces ```text @@ -100,7 +100,7 @@ If a host does not properly implement half-close you may see an error message `[ This has been a [known issue](https://github.com/docker/libnetwork/issues/1204) in Docker, but may manifest in other environments as well. -## ACL Not Found +## ACL not found ```text RPC error making call: rpc error making call: ACL not found @@ -108,7 +108,9 @@ RPC error making call: rpc error making call: ACL not found This indicates that you have ACL enabled in your cluster, but you aren't passing a valid token. Make sure that when creating your tokens that they have the correct permissions set. In addition, you would want to make sure that an agent token is provided on each call. -## TLS and Certificates +## TLS and certificates + +The follow errors are related to TLS and certificate issues. ### Incorrect certificate or certificate name @@ -150,8 +152,20 @@ You have installed an Enterprise version of Consul. If you are an Enterprise cus -> **Note:** Enterprise binaries can be identified on our [download site][releases] by the `+ent` suffix. +## Rate limit reached on the server + +You may receive a `RESOURCE_EXHAUSTED` error from the Consul server if the maximum number of read or write requests per second have been reached. Refer to [Set a global limit on traffic rates](/consul/docs/agent/set-global-traffic-rate-limits) for additional information. You can retry another server unless the number of retries is exhausted. If the number of retries is exhausted, you should implement an exponential backoff. + +The RESOURCE_EXHAUSTED RPC response is translated into a `429 Too Many Requests` error code on the HTTP interface. + +The server may respond as `UNAVAILABLE` if it is the leader node and the global write request rate limit is reached. The solution is to apply an exponential backoff until the leader has capacity to serve those requests. + +The `UNAVAILABLE` RPC response is translated into a `503 Service Unavailable` error code on the RPC requests sent through HTTP interface. + ## Common errors on Kubernetes +The following error messages are specific to Kubernetes issues. + ### Unable to connect to the Consul client on the same host If the pods are unable to connect to a Consul client running on the same host, From dc375c7cc30de628a419e2791e8fcaeaba0b5c65 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 14 Apr 2023 17:03:45 -0400 Subject: [PATCH 167/421] backport of commit cd4d749ede49f89ac16b5bb87bbe0be8c1704ad1 (#17010) Co-authored-by: trujillo-adam --- website/content/docs/troubleshoot/common-errors.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/troubleshoot/common-errors.mdx b/website/content/docs/troubleshoot/common-errors.mdx index 3c5ff5b076ad..4c19063375b7 100644 --- a/website/content/docs/troubleshoot/common-errors.mdx +++ b/website/content/docs/troubleshoot/common-errors.mdx @@ -154,7 +154,7 @@ You have installed an Enterprise version of Consul. If you are an Enterprise cus ## Rate limit reached on the server -You may receive a `RESOURCE_EXHAUSTED` error from the Consul server if the maximum number of read or write requests per second have been reached. Refer to [Set a global limit on traffic rates](/consul/docs/agent/set-global-traffic-rate-limits) for additional information. You can retry another server unless the number of retries is exhausted. If the number of retries is exhausted, you should implement an exponential backoff. +You may receive a `RESOURCE_EXHAUSTED` error from the Consul server if the maximum number of read or write requests per second have been reached. Refer to [Set a global limit on traffic rates](/consul/docs/agent/limits/set-global-traffic-rate-limits) for additional information. You can retry another server unless the number of retries is exhausted. If the number of retries is exhausted, you should implement an exponential backoff. The RESOURCE_EXHAUSTED RPC response is translated into a `429 Too Many Requests` error code on the HTTP interface. From 9b07a920b1f84a9bf966df65ee4b84cb55812d61 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 17 Apr 2023 14:41:33 -0400 Subject: [PATCH 168/421] Backport of added an intro statement for the SI conf entry confiration model into release/1.15.x (#17018) * backport of commit b4b8e7b114061045310bf9567717bb756404d8a7 * backport of commit d24b7e4d032b9587455604dc54e7cfa6eb635e90 --------- Co-authored-by: trujillo-adam --- .../docs/connect/config-entries/service-intentions.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/content/docs/connect/config-entries/service-intentions.mdx b/website/content/docs/connect/config-entries/service-intentions.mdx index 1eb2c42667a0..40fc94c0bb0d 100644 --- a/website/content/docs/connect/config-entries/service-intentions.mdx +++ b/website/content/docs/connect/config-entries/service-intentions.mdx @@ -11,6 +11,8 @@ This topic provides reference information for the service intentions configurati ## Configuration model +The following outline shows how to format the service intentions configuration entry. Click on a property name to view details about the configuration. + @@ -90,6 +92,8 @@ This topic provides reference information for the service intentions configurati ## Complete configuration +When every field is defined, a service intentions configuration entry has the following form: + @@ -278,7 +282,7 @@ spec: ## Specification -This section provides details about the fields you can configure in the service defaults configuration entry. +This section provides details about the fields you can configure in the service intentions configuration entry. From df8df9b8944b2e0e7fc1fce070336706b3219e10 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 17 Apr 2023 19:54:27 -0400 Subject: [PATCH 169/421] Backport of docs: update docs related to GH-16779 into release/1.15.x (#17021) * backport of commit c030771c774929e99c59d8ebc4bde2c7ed0f5fa9 * backport of commit a12af289cb096f0ad7309a1f7cf76328eae3352b * backport of commit a4e963b133ae4e2575bd2278ae7ad203c46a4a1a --------- Co-authored-by: Jared Kirschner Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> --- CHANGELOG.md | 2 +- .../docs/release-notes/consul/v1_15_x.mdx | 16 +++++++--------- .../content/docs/upgrading/upgrade-specific.mdx | 13 ++++--------- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b5259a054ee..5a306dc2ddc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ BUG FIXES: KNOWN ISSUES: -* connect: An issue with leaf certificate rotation can cause some service instances to lose their ability to communicate in the mesh after 72 hours (LeafCertTTL). This issue is not consistently reproducible. We are working to address this issue in an upcoming patch release. To err on the side of caution, service mesh deployments should not upgrade to Consul v1.15 at this time. Refer to [[GH-16779](https://github.com/hashicorp/consul/issues/16779)] for the latest information. +* connect: A race condition can cause some service instances to lose their ability to communicate in the mesh after 72 hours (LeafCertTTL) due to a problem with leaf certificate rotation. This bug is fixed in Consul v1.15.2 by [GH-16818](https://github.com/hashicorp/consul/issues/16818). BREAKING CHANGES: diff --git a/website/content/docs/release-notes/consul/v1_15_x.mdx b/website/content/docs/release-notes/consul/v1_15_x.mdx index 99c2961bd4ab..d90f918b6373 100644 --- a/website/content/docs/release-notes/consul/v1_15_x.mdx +++ b/website/content/docs/release-notes/consul/v1_15_x.mdx @@ -68,14 +68,11 @@ For more detailed information, please refer to the [upgrade details page](/consu The following issues are known to exist in the v1.15.x releases: -- All current 1.15.x versions are under investigation for a not-consistently-reproducible - issue that can cause some service instances to lose their ability to communicate in the mesh after +- v1.15.0 - v1.15.1 contain a race condition that can cause + some service instances to lose their ability to communicate in the mesh after [72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) due to a problem with leaf certificate rotation. - We will update this section with more information as our investigation continues, - including the target availability for a fix. - Refer to [GH-16779](https://github.com/hashicorp/consul/issues/16779) - for the latest information. + This is resolved in Consul v1.15.2. - For v1.15.0, Consul is reporting newer releases of Envoy (for example, v1.25.1) as not supported, even though these versions are listed as valid in the [Envoy compatilibity matrix](/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent). The following error would result for newer versions of Envoy: @@ -83,15 +80,16 @@ The following issues are known to exist in the v1.15.x releases: Envoy version 1.25.1 is not supported. If there is a reason you need to use this version of envoy use the ignore-envoy-compatibility flag. Using an unsupported version of Envoy is not recommended and your experience may vary. ``` - The workaround to resolve this issue until Consul v1.15.1 would be to run the client agents with the new `ingore-envoy-compatiblity` flag: + To workaround this issue on Consul v1.15.0, launch sidecar proxies + with the `ignore-envoy-compatiblity` flag: ```shell-session $ consul connect envoy --ignore-envoy-compatibility ``` -- For v1.15.0, there is a known issue where `consul acl token read -self` requires an `-accessor-id`. This is resolved in the uppcoming Consul v1.15.1 patch release. +- For v1.15.0, there is a known issue where `consul acl token read -self` requires an `-accessor-id`. This is resolved in Consul v1.15.1. -- For v1.15.0, there is a known issue where search filters produced errors and resulted in lists not showing full results until being interacted with. This is resolved in the upcoming Consul v1.15.1 patch release. +- For v1.15.0, there is a known issue where search filters produced errors and resulted in lists not showing full results until being interacted with. This is resolved in Consul v1.15.1. ## Changelogs diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 1c9a73849107..39a15d2f1f27 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -16,21 +16,16 @@ upgrade flow. ## Consul 1.15.x -#### Service mesh known issue +#### Service mesh compatibility ((#service-mesh-compatibility-1-15)) -To err on the side of caution, -service mesh deployments should not upgrade to Consul v1.15 at this time. +Upgrade to **Consul version 1.15.2 or later**. -We are currently investigating a not-consistently-reproducible issue that can cause +Consul versions 1.15.0 - 1.15.1 contain a race condition that can cause some service instances to lose their ability to communicate in the mesh after [72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) due to a problem with leaf certificate rotation. -We will update this section with more information as our investigation continues, -including the target availability for a fix. -If you are already operating Consul v1.15, refer to discussion of this issue on -[GH-16779](https://github.com/hashicorp/consul/issues/16779) -for potential workarounds and to share your observations. +This bug is fixed in Consul versions 1.15.2 and newer. #### Removing configuration options From ac96ab8ba18a8fdca67daaaf3c3c5d50a0df03b2 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 18 Apr 2023 13:59:59 -0400 Subject: [PATCH 170/421] backport of commit 12d5cc0c377c5a37ff75112227247cb1271157fa (#17029) Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --- agent/hcp/manager_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/hcp/manager_test.go b/agent/hcp/manager_test.go index cb4d729b7fa8..2d6cfa66b474 100644 --- a/agent/hcp/manager_test.go +++ b/agent/hcp/manager_test.go @@ -36,7 +36,6 @@ func TestManager_Run(t *testing.T) { // Make sure after manager has stopped no more statuses are pushed. cancel() - mgr.SendUpdate() client.AssertExpectations(t) } From c205bd086110f026440296aec186e08ad1933e3a Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Tue, 18 Apr 2023 14:15:20 -0400 Subject: [PATCH 171/421] Backport of Bump the golang.org/x/net to 0.7.0 to address CVE-2022-41723 into release/1.15.x (#17027) * backport of commit 0561b249604681e60baad92ea45d904868a4be1e * backport of commit 7d6bd816f1b688ccf800d6b081c0e9b1f7acbb1c --------- Co-authored-by: Kevin Wang Co-authored-by: Nathan Coleman --- .changelog/16754.txt | 3 +++ go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 .changelog/16754.txt diff --git a/.changelog/16754.txt b/.changelog/16754.txt new file mode 100644 index 000000000000..fc2abc9ebfd8 --- /dev/null +++ b/.changelog/16754.txt @@ -0,0 +1,3 @@ +```release-note:security +Upgrade golang.org/x/net to address [CVE-2022-41723](https://nvd.nist.gov/vuln/detail/CVE-2022-41723) +``` diff --git a/go.mod b/go.mod index 0a525cbc8e50..8e1a59773c05 100644 --- a/go.mod +++ b/go.mod @@ -93,10 +93,10 @@ require ( go.etcd.io/bbolt v1.3.6 go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/net v0.4.0 + golang.org/x/net v0.7.0 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.3.0 + golang.org/x/sys v0.5.0 golang.org/x/time v0.3.0 google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737 google.golang.org/grpc v1.49.0 @@ -226,8 +226,8 @@ require ( go.uber.org/atomic v1.9.0 // indirect golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/api v0.57.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 461cd0136ff0..15c7db3b7949 100644 --- a/go.sum +++ b/go.sum @@ -1188,8 +1188,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1316,13 +1316,13 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1334,8 +1334,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From d352a453923ca0a50086414035dde5264417217a Mon Sep 17 00:00:00 2001 From: John Murret Date: Tue, 18 Apr 2023 21:45:20 -0600 Subject: [PATCH 172/421] ci: add test-integrations (#16915) (#17041) * add test-integrations workflow * add test-integrations success job * update vault integration testing versions (#16949) * change parallelism to 4 forgotestsum. use env.CONSUL_VERSION so we can see the version. * use env for repeated values * match test to circleci * fix envvar * fix envvar 2 * fix envvar 3 * fix envvar 4 * fix envvar 5 * make upgrade and compatibility tests match circleci * run go env to check environment * debug docker * debug docker * revert debug docker * going back to command that worked 5 days ago for compatibility tests * Update Envoy versions to reflect changes in #16889 * cd to test dir * try running ubuntu latest * update PR with latest changes that work in enterprise * yaml still sucks * test GH fix (localhost resolution) * change for testing * test splitting and ipv6 lookup for compatibility and upgrade tests * fix indention * consul as image name * remove the on push * add gotestsum back in * removing the use of the gotestsum download action * yaml sucks today just like yesterday * fixing nomad tests * worked out the kinks on enterprise --------- Signed-off-by: Dan Bond Co-authored-by: John Eikenberry Co-authored-by: Dan Bond Co-authored-by: Nathan Coleman Co-authored-by: Sarah --- .circleci/config.yml | 2 + .github/workflows/test-integrations.yml | 501 ++++++++++++++++++++++++ 2 files changed, 503 insertions(+) create mode 100644 .github/workflows/test-integrations.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e2a1832ec05..9fe7404bbc52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -347,6 +347,7 @@ jobs: sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz environment: <<: *ENVIRONMENT + - run: go env - run: *install-gotestsum - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: @@ -410,6 +411,7 @@ jobs: sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz environment: <<: *ENVIRONMENT + - run: go env - run: *install-gotestsum - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml new file mode 100644 index 000000000000..6ba474643382 --- /dev/null +++ b/.github/workflows/test-integrations.yml @@ -0,0 +1,501 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: test-integrations + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branch prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.9.0" + CONSUL_BINARY_UPLOAD_NAME: consul-bin + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }} + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - id: runners + run: .github/scripts/get_runner_classes.sh + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + nomad-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] + steps: + - name: Checkout Nomad + uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + with: + repository: hashicorp/nomad + ref: ${{ matrix.nomad-version }} + + - name: Install Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Fetch Consul binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: Restore Consul permissions + run: | + chmod +x ./bin/consul + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Make Nomad dev build + run: make pkg/linux_amd64/nomad + + - name: Run integration tests + run: | + go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \ + gotestsum \ + --format=short-verbose \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="./command/agent/consul" \ + --junitfile $TEST_RESULTS_DIR/results.xml -- \ + -run TestConsul + + vault-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] + env: + VAULT_BINARY_VERSION: ${{ matrix.vault-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Install Vault + run: | + wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip" + unzip -d /tmp /tmp/vault.zip + echo "/tmp" >> $GITHUB_PATH + + - name: Run Connect CA Provider Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca + # Run leader tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul + # Run agent tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 3 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + needs: + - setup + - generate-envoy-job-matrices + - dev-build + strategy: + fail-fast: false + matrix: + envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/aws arn | AWS_ROLE_ARN ; + + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: container-logs + path: ./test/integration/connect/envoy/workdir/logs + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + generate-compatibility-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Compatibility Job Matrices + outputs: + compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Compatibility Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 5 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container + { + echo -n "compatibility-matrix=" + find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ + | grep -v util | grep -v upgrade \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join(" "))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + compatibility-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - dev-build + - generate-compatibility-job-matrices + strategy: + fail-fast: false + matrix: + test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + + # Build the consul:local image from the already built binary + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Compatibility Integration Tests + run: | + mkdir -p "/tmp/test-results" + cd ./test/integration/consul-container + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + "${{ matrix.test-cases }}" \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version latest + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + generate-upgrade-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Upgrade Job Matrices + outputs: + upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - name: Generate Updgrade Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 4 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container/test/upgrade + { + echo -n "upgrade-matrix=" + go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + upgrade-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + needs: + - setup + - dev-build + - generate-upgrade-job-matrices + strategy: + fail-fast: false + matrix: + consul-version: [ "1.14", "1.15"] + test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }} + env: + CONSUL_VERSION: ${{ matrix.consul-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; + kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; + + # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS. + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 + with: + username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} + + + # Get go binary from workspace + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Upgrade Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container/test/upgrade + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json ./... \ + -run "${{ matrix.test-cases }}" \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version "${{ env.CONSUL_VERSION }}" + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + test-integrations-success: + needs: + - setup + - dev-build + - nomad-integration-test + - vault-integration-test + - generate-envoy-job-matrices + - envoy-integration-test + - generate-compatibility-job-matrices + - compatibility-integration-test + - generate-upgrade-job-matrices + - upgrade-integration-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "test-integrations succeeded" From ee82e1d01e3a8d6d37f2a3faaa648671820c3844 Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 19 Apr 2023 11:28:59 -0600 Subject: [PATCH 173/421] ci: remove test-integrations CircleCI workflow (#16928) (#17049) * remove all CircleCI files * remove references to CircleCI * remove more references to CircleCI * pin golangci-lint to v1.51.1 instead of v1.51 --- .circleci/bash_env.sh | 10 - .circleci/config.yml | 601 ------------------ .circleci/scripts/rerun-fails-report.sh | 21 - .../terraform/load-test/.terraform.lock.hcl | 110 ---- .circleci/terraform/load-test/main.tf | 26 - .circleci/terraform/load-test/variables.tf | 30 - .github/CONTRIBUTING.md | 3 +- .github/pr-labeler.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/nightly-test-1.11.x.yaml | 4 +- .github/workflows/nightly-test-1.12.x.yaml | 4 +- .github/workflows/nightly-test-1.13.x.yaml | 4 +- .github/workflows/nightly-test-1.14.x.yaml | 4 +- .github/workflows/nightly-test-main.yaml | 4 +- .github/workflows/reusable-lint.yml | 2 +- .github/workflows/test-integrations.yml | 4 +- GNUmakefile | 12 +- agent/checks/check_test.go | 3 - agent/connect/ca/testing.go | 2 +- docs/README.md | 3 +- test/integration/connect/envoy/run-tests.sh | 10 +- test/load/README.md | 3 - ui/packages/consul-ui/docs/upgrades.mdx | 2 +- 23 files changed, 22 insertions(+), 844 deletions(-) delete mode 100644 .circleci/bash_env.sh delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/scripts/rerun-fails-report.sh delete mode 100755 .circleci/terraform/load-test/.terraform.lock.hcl delete mode 100644 .circleci/terraform/load-test/main.tf delete mode 100644 .circleci/terraform/load-test/variables.tf diff --git a/.circleci/bash_env.sh b/.circleci/bash_env.sh deleted file mode 100644 index 38bcfd5bd367..000000000000 --- a/.circleci/bash_env.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -export GIT_COMMIT=$(git rev-parse --short HEAD) -export GIT_COMMIT_YEAR=$(git show -s --format=%cd --date=format:%Y HEAD) -export GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true) -export GIT_IMPORT=github.com/hashicorp/consul/version -# we're using this for build date because it's stable across platform builds -# the env -i and -noprofile are used to ensure we don't try to recursively call this profile when starting bash -export GIT_DATE=$(env -i /bin/bash --noprofile -norc /home/circleci/project/build-support/scripts/build-date.sh) -export GOLDFLAGS="-X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X ${GIT_IMPORT}.BuildDate=${GIT_DATE}" diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 9fe7404bbc52..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,601 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - ---- -version: 2.1 - -parameters: - commit: - type: string - default: "" - description: "Commit to run load tests against" - -references: - paths: - test-results: &TEST_RESULTS_DIR /tmp/test-results - environment: &ENVIRONMENT - TEST_RESULTS_DIR: *TEST_RESULTS_DIR - EMAIL: noreply@hashicorp.com - GIT_AUTHOR_NAME: circleci-consul - GIT_COMMITTER_NAME: circleci-consul - S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 - BASH_ENV: .circleci/bash_env.sh - GO_VERSION: 1.20.1 - envoy-versions: &supported_envoy_versions - - &default_envoy_version "1.22.11" - - "1.23.8" - - "1.24.6" - - "1.25.4" - nomad-versions: &supported_nomad_versions - - &default_nomad_version "1.3.3" - - "1.2.10" - - "1.1.16" - vault-versions: &supported_vault_versions - - &default_vault_version "1.12.2" - - "1.11.6" - - "1.10.9" - - "1.9.10" - consul-versions: &consul_versions - - "1.13" - - "1.14" - - "1.15" - images: - # When updating the Go version, remember to also update the versions in the - # workflows section for go-test-lib jobs. - go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.20.1 - ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:16-browsers - ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02 - cache: - yarn: &YARN_CACHE_KEY consul-ui-v9-{{ checksum "ui/yarn.lock" }} - -steps: - install-gotestsum: &install-gotestsum - name: install gotestsum - environment: - GOTESTSUM_RELEASE: 1.9.0 - command: | - ARCH=`uname -m` - if [[ "$ARCH" == "aarch64" ]]; then - ARCH="arm64" - else - ARCH="amd64" - fi - url=https://github.com/gotestyourself/gotestsum/releases/download - curl -sSL "${url}/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_${ARCH}.tar.gz" | \ - sudo tar -xz --overwrite -C /usr/local/bin gotestsum - - get-aws-cli: &get-aws-cli - run: - name: download and install AWS CLI - command: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - echo -e "${AWS_CLI_GPG_KEY}" | gpg --import - curl -o awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig - gpg --verify awscliv2.sig awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install - - # This step MUST be at the end of any set of steps due to the 'when' condition - notify-slack-failure: ¬ify-slack-failure - name: notify-slack-failure - when: on_fail - command: | - if [[ $CIRCLE_BRANCH == "main" ]]; then - CIRCLE_ENDPOINT="https://app.circleci.com/pipelines/github/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}?branch=${CIRCLE_BRANCH}" - GITHUB_ENDPOINT="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}" - COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1) - SHORT_REF=$(git rev-parse --short "${CIRCLE_SHA1}") - curl -X POST -H 'Content-type: application/json' \ - --data \ - "{ \ - \"attachments\": [ \ - { \ - \"fallback\": \"CircleCI job failed!\", \ - \"text\": \"❌ Failed: \`${CIRCLE_USERNAME}\`'s <${CIRCLE_BUILD_URL}|${CIRCLE_STAGE}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${CIRCLE_BRANCH}\`!\n\n- <${COMMIT_MESSAGE}\", \ - \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \ - \"ts\": \"$(date +%s)\", \ - \"color\": \"danger\" \ - } \ - ] \ - }" "${FEED_CONSUL_GH_URL}" - else - echo "Not posting slack failure notifications for non-main branch" - fi - -commands: - assume-role: - description: "Assume role to an ARN" - parameters: - access-key: - type: env_var_name - default: AWS_ACCESS_KEY_ID - secret-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - role-arn: - type: env_var_name - default: ROLE_ARN - steps: - # Only run the assume-role command for the main repo. The AWS credentials aren't available for forks. - - run: | - if [[ "${CIRCLE_BRANCH%%/*}/" != "pull/" ]]; then - export AWS_ACCESS_KEY_ID="${<< parameters.access-key >>}" - export AWS_SECRET_ACCESS_KEY="${<< parameters.secret-key >>}" - export ROLE_ARN="${<< parameters.role-arn >>}" - # assume role has duration of 15 min (the minimum allowed) - CREDENTIALS="$(aws sts assume-role --duration-seconds 900 --role-arn ${ROLE_ARN} --role-session-name build-${CIRCLE_SHA1} | jq '.Credentials')" - echo "export AWS_ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.AccessKeyId')" >> $BASH_ENV - echo "export AWS_SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.SecretAccessKey')" >> $BASH_ENV - echo "export AWS_SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.SessionToken')" >> $BASH_ENV - fi - - run-go-test-full: - parameters: - go_test_flags: - type: string - default: "" - steps: - - attach_workspace: - at: /home/circleci/go/bin - - run: go mod download - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname) - echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" - echo $PACKAGE_NAMES - # some tests expect this umask, and arm images have a different default - umask 0022 - - << parameters.go_test_flags >> - - gotestsum \ - --format=short-verbose \ - --jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \ - --debug \ - --rerun-fails=3 \ - --rerun-fails-max-failures=40 \ - --rerun-fails-report=/tmp/gotestsum-rerun-fails \ - --packages="$PACKAGE_NAMES" \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - ${GO_TEST_FLAGS-} \ - -cover -coverprofile=coverage.txt - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: &rerun-fails-report - name: "Re-run fails report" - command: | - .circleci/scripts/rerun-fails-report.sh /tmp/gotestsum-rerun-fails - - run: *notify-slack-failure - -jobs: - check-go-mod: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - steps: - - checkout - - run: go mod tidy - - run: | - if [[ -n $(git status -s) ]]; then - echo "Git directory has changes" - git status -s - exit 1 - fi - - run: *notify-slack-failure - - # build is a templated job for build-x - build-distros: &build-distros - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: &build-env - <<: *ENVIRONMENT - steps: - - checkout - - run: - name: Build - command: | - for os in $XC_OS; do - target="./pkg/bin/${GOOS}_${GOARCH}/" - GOOS="$os" CGO_ENABLED=0 go build -o "${target}" -ldflags "${GOLDFLAGS}" -tags "${GOTAGS}" - done - - # save dev build to CircleCI - - store_artifacts: - path: ./pkg/bin - - run: *notify-slack-failure - - # create a development build - dev-build: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - steps: - - checkout - - attach_workspace: # this normally runs as the first job and has nothing to attach; only used in main branch after rebuilding UI - at: . - - run: - name: Build - command: | - make dev - mkdir -p /home/circleci/go/bin - cp ./bin/consul /home/circleci/go/bin/consul - - # save dev build to pass to downstream jobs - - persist_to_workspace: - root: /home/circleci/go/bin - paths: - - consul - - run: *notify-slack-failure - - # upload development build to s3 - dev-upload-s3: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - steps: - - checkout - - *get-aws-cli - - assume-role: - access-key: AWS_ACCESS_KEY_ID_S3_UPLOAD - secret-key: AWS_SECRET_ACCESS_KEY_S3_UPLOAD - role-arn: ROLE_ARN_S3_UPLOAD - # get consul binary - - attach_workspace: - at: bin/ - - run: - name: package binary - command: zip -j consul.zip bin/consul - - run: - name: Upload to s3 - command: | - if [ -n "${S3_ARTIFACT_PATH}" ]; then - aws s3 cp \ - --metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \ - "consul.zip" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.zip" --acl public-read - else - echo "CircleCI - S3_ARTIFACT_PATH was not set" - exit 1 - fi - - run: *notify-slack-failure - - # upload dev docker image - dev-upload-docker: - docker: - - image: *GOLANG_IMAGE # use a circleci image so the attach_workspace step works (has ca-certs installed) - environment: - <<: *ENVIRONMENT - steps: - - checkout - # get consul binary - - attach_workspace: - at: bin/ - - setup_remote_docker - - run: make ci.dev-docker - - run: *notify-slack-failure - nomad-integration-test: &NOMAD_TESTS - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.20 - parameters: - nomad-version: - type: enum - enum: *supported_nomad_versions - default: *default_nomad_version - environment: - <<: *ENVIRONMENT - NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /home/circleci/go/src/github.com/hashicorp/nomad - NOMAD_VERSION: << parameters.nomad-version >> - steps: &NOMAD_INTEGRATION_TEST_STEPS - - run: git clone https://github.com/hashicorp/nomad.git --branch v${NOMAD_VERSION} ${NOMAD_WORKING_DIR} - - # get consul binary - - attach_workspace: - at: /home/circleci/go/bin - - # make dev build of nomad - - run: - command: make pkg/linux_amd64/nomad - working_directory: *NOMAD_WORKING_DIR - - - run: *install-gotestsum - - # run integration tests - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR - gotestsum \ - --format=short-verbose \ - --junitfile $TEST_RESULTS_DIR/results.xml -- \ - ./command/agent/consul -run TestConsul - working_directory: *NOMAD_WORKING_DIR - - # store test results for CircleCI - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - compatibility-integration-test: - machine: - image: *UBUNTU_CI_IMAGE - docker_layer_caching: true - parallelism: 1 - steps: - - checkout - # Get go binary from workspace - - attach_workspace: - at: . - # Build the consul:local image from the already built binary - - run: - command: | - sudo rm -rf /usr/local/go - wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz - sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz - environment: - <<: *ENVIRONMENT - - run: go env - - run: *install-gotestsum - - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - - run: - name: Compatibility Integration Tests - command: | - mkdir -p /tmp/test-results/ - cd ./test/integration/consul-container - docker run --rm consul:local consul version - gotestsum \ - --raw-command \ - --format=short-verbose \ - --debug \ - --rerun-fails=3 \ - --packages="./..." \ - -- \ - go test \ - -p=4 \ - -timeout=30m \ - -json \ - `go list ./... | grep -v upgrade` \ - --target-image consul \ - --target-version local \ - --latest-image consul \ - --latest-version latest - ls -lrt - environment: - # this is needed because of incompatibility between RYUK container and circleci - GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - # tput complains if this isn't set to something. - TERM: ansi - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - upgrade-integration-test: - machine: - image: *UBUNTU_CI_IMAGE - docker_layer_caching: true - parallelism: 3 - resource_class: large - parameters: - consul-version: - type: enum - enum: *consul_versions - environment: - CONSUL_VERSION: << parameters.consul-version >> - steps: - - checkout - # Get go binary from workspace - - attach_workspace: - at: . - # Build the consul:local image from the already built binary - - run: - command: | - sudo rm -rf /usr/local/go - wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz - sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz - environment: - <<: *ENVIRONMENT - - run: go env - - run: *install-gotestsum - - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - - run: - name: Upgrade Integration Tests - command: | - mkdir -p /tmp/test-results/ - cd ./test/integration/consul-container - docker run --rm consul:local consul version - gotestsum \ - --raw-command \ - --format=short-verbose \ - --debug \ - --rerun-fails=3 \ - --packages="./..." \ - -- \ - go test \ - -p=4 \ - -tags "${GOTAGS}" \ - -timeout=30m \ - -json \ - ./.../upgrade/ \ - --target-image consul \ - --target-version local \ - --latest-image consul \ - --latest-version $CONSUL_VERSION - ls -lrt - environment: - # this is needed because of incompatibility between RYUK container and circleci - GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - # tput complains if this isn't set to something. - TERM: ansi - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - envoy-integration-test: &ENVOY_TESTS - machine: - image: *UBUNTU_CI_IMAGE - parallelism: 4 - resource_class: medium - parameters: - envoy-version: - type: enum - enum: *supported_envoy_versions - default: *default_envoy_version - xds-target: - type: enum - enum: ["server", "client"] - default: "server" - environment: - ENVOY_VERSION: << parameters.envoy-version >> - XDS_TARGET: << parameters.xds-target >> - AWS_LAMBDA_REGION: us-west-2 - steps: &ENVOY_INTEGRATION_TEST_STEPS - - checkout - - assume-role: - access-key: AWS_ACCESS_KEY_ID_LAMBDA - secret-key: AWS_SECRET_ACCESS_KEY_LAMBDA - role-arn: ROLE_ARN_LAMBDA - # Get go binary from workspace - - attach_workspace: - at: . - - run: *install-gotestsum - # Build the consul:local image from the already built binary - - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - - run: - name: Envoy Integration Tests - command: | - subtests=$(ls -d test/integration/connect/envoy/*/ | xargs -n 1 basename | circleci tests split) - echo "Running $(echo $subtests | wc -w) subtests" - echo "$subtests" - subtests_pipe_sepr=$(echo "$subtests" | xargs | sed 's/ /|/g') - mkdir -p /tmp/test-results/ - gotestsum -- -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/($subtests_pipe_sepr)" - environment: - GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - LAMBDA_TESTS_ENABLED: "true" - # tput complains if this isn't set to something. - TERM: ansi - - store_artifacts: - path: ./test/integration/connect/envoy/workdir/logs - destination: container-logs - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - # run integration tests for the connect ca providers with vault - vault-integration-test: - docker: - - image: *GOLANG_IMAGE - parameters: - vault-version: - type: enum - enum: *supported_vault_versions - default: *default_vault_version - environment: - <<: *ENVIRONMENT - VAULT_BINARY_VERSION: << parameters.vault-version >> - steps: &VAULT_INTEGRATION_TEST_STEPS - - run: - name: Install vault - command: | - wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/${VAULT_BINARY_VERSION}/vault_${VAULT_BINARY_VERSION}_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - vault version - - checkout - - run: go mod download - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR - make test-connect-ca-providers - - store_test_results: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - # The noop job is a used as a very fast job in the verify-ci workflow because every workflow - # requires at least one job. It does nothing. - noop: - docker: - - image: docker.mirror.hashicorp.services/alpine:latest - steps: - - run: "echo ok" - -workflows: - test-integrations: - jobs: - - dev-build: &filter-ignore-non-go-branches - filters: - branches: - ignore: - - stable-website - - /^docs\/.*/ - - /^ui\/.*/ - - /^mktg-.*/ # Digital Team Terraform-generated branches' prefix - - /^backport\/docs\/.*/ - - /^backport\/ui\/.*/ - - /^backport\/mktg-.*/ - - dev-upload-s3: &dev-upload - requires: - - dev-build - filters: - branches: - ignore: - - /^pull\/.*$/ # only push dev builds from non forks - - main # all main dev uploads will include a UI rebuild in build-distros - - dev-upload-docker: - <<: *dev-upload - context: consul-ci - - nomad-integration-test: - requires: - - dev-build - matrix: - parameters: - nomad-version: *supported_nomad_versions - - vault-integration-test: - matrix: - parameters: - vault-version: *supported_vault_versions - <<: *filter-ignore-non-go-branches - - envoy-integration-test: - requires: - - dev-build - matrix: - parameters: - envoy-version: *supported_envoy_versions - xds-target: ["server", "client"] - - compatibility-integration-test: - requires: - - dev-build - - upgrade-integration-test: - requires: - - dev-build - matrix: - parameters: - consul-version: *consul_versions - - - noop diff --git a/.circleci/scripts/rerun-fails-report.sh b/.circleci/scripts/rerun-fails-report.sh deleted file mode 100755 index 959f0708e685..000000000000 --- a/.circleci/scripts/rerun-fails-report.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Add a comment on the github PR if there were any rerun tests. -# -set -eu -o pipefail - -report_filename="${1?report filename is required}" -if [ ! -s "$report_filename" ]; then - echo "gotestsum rerun report file is empty or missing" - exit 0 -fi - -function report { - echo ":repeat: gotestsum re-ran some tests in $CIRCLE_BUILD_URL" - echo - echo '```' - cat "$report_filename" - echo '```' -} - -report \ No newline at end of file diff --git a/.circleci/terraform/load-test/.terraform.lock.hcl b/.circleci/terraform/load-test/.terraform.lock.hcl deleted file mode 100755 index 9f4d960dcd42..000000000000 --- a/.circleci/terraform/load-test/.terraform.lock.hcl +++ /dev/null @@ -1,110 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "3.41.0" - constraints = "~> 3.0, >= 3.27.0" - hashes = [ - "h1:YLbsjPt/oZdEhV+KJzMVBwGDViw14Ih5bYr+EOudIVw=", - "zh:01449ed390710428c92dcd3c6b8ba7e06cc1581b927e96eabe9ebc2653d1e3e0", - "zh:259c1267ab5798e90c8edb4b9c3b17c1dd98e5265c121eaf025a5836e88f4d1d", - "zh:2671ec766eb63d642b8b3d847d67db83d578a44d4945bc45ddd7fbb6d09298ca", - "zh:36082943070c8f4f9a1e557a6b18d279db079b19210cd5249ba03c87de44e5d4", - "zh:49a52c679a14c7755db34e0b98ef062f5e42b7beec1decec2511ecef16690b3f", - "zh:82cf0db34865d8844139a6db35436a6b4664995972dc53e298c93a7133101b0f", - "zh:9082239ae74e4f8b9763087bf454dcfb1019e1a65c4d9ab8057f8425b9da550b", - "zh:a9b51d299b3ffe07684e86d8ea11513411f53375439be5aa87fdfef59cbe5dfa", - "zh:b33fb3990c9bb2a1337725651a98d9563a3b91b50ddeb7c7b655c463faa81dda", - "zh:bd759da1e0c18a2c17bfe607660d52d8981aa51460d70d2e338ddbcef1b50183", - "zh:eebb98f9ba764dd09b059c5865ce7e8bace49fe470980f813a767cbe833a933e", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" - hashes = [ - "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.1.0" - hashes = [ - "h1:xhbHC6in3nQryvTQBWKxebi3inG5OCgHgc4fRxL0ymc=", - "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", - "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", - "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", - "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", - "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", - "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", - "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", - "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", - "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", - "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", - "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" - hashes = [ - "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "3.1.0" - hashes = [ - "h1:XTU9f6sGMZHOT8r/+LWCz2BZOPH127FBTPjMMEAAu1U=", - "zh:3d46616b41fea215566f4a957b6d3a1aa43f1f75c26776d72a98bdba79439db6", - "zh:623a203817a6dafa86f1b4141b645159e07ec418c82fe40acd4d2a27543cbaa2", - "zh:668217e78b210a6572e7b0ecb4134a6781cc4d738f4f5d09eb756085b082592e", - "zh:95354df03710691773c8f50a32e31fca25f124b7f3d6078265fdf3c4e1384dca", - "zh:9f97ab190380430d57392303e3f36f4f7835c74ea83276baa98d6b9a997c3698", - "zh:a16f0bab665f8d933e95ca055b9c8d5707f1a0dd8c8ecca6c13091f40dc1e99d", - "zh:be274d5008c24dc0d6540c19e22dbb31ee6bfdd0b2cddd4d97f3cd8a8d657841", - "zh:d5faa9dce0a5fc9d26b2463cea5be35f8586ab75030e7fa4d4920cd73ee26989", - "zh:e9b672210b7fb410780e7b429975adcc76dd557738ecc7c890ea18942eb321a5", - "zh:eb1f8368573d2370605d6dbf60f9aaa5b64e55741d96b5fb026dbfe91de67c0d", - "zh:fc1e12b713837b85daf6c3bb703d7795eaf1c5177aebae1afcf811dd7009f4b0", - ] -} diff --git a/.circleci/terraform/load-test/main.tf b/.circleci/terraform/load-test/main.tf deleted file mode 100644 index 1a8865c065d9..000000000000 --- a/.circleci/terraform/load-test/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - backend "s3" { - } -} - -provider "aws" { - assume_role { - role_arn = var.role_arn - } -} - -module "load-test" { - source = "../../../test/load/terraform" - - vpc_az = ["us-east-2a", "us-east-2b"] - vpc_name = var.vpc_name - vpc_cidr = "10.0.0.0/16" - vpc_allwed_ssh_cidr = "0.0.0.0/0" - public_subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24"] - private_subnet_cidrs = ["10.0.3.0/24"] - test_public_ip = true - ami_owners = var.ami_owners - consul_download_url = var.consul_download_url - cluster_name = var.cluster_name - cluster_tag_key = var.cluster_tag_key -} diff --git a/.circleci/terraform/load-test/variables.tf b/.circleci/terraform/load-test/variables.tf deleted file mode 100644 index 414cfa84e7fe..000000000000 --- a/.circleci/terraform/load-test/variables.tf +++ /dev/null @@ -1,30 +0,0 @@ -variable "vpc_name" { - description = "Name of the VPC" -} - -variable "ami_owners" { - type = list(string) - description = "The account owner number which the desired AMI is in" -} - -variable "role_arn" { - type = string - description = "Role ARN for assume role" -} - -variable "consul_download_url" { - type = string - description = "URL to download the Consul binary from" - default = "" -} -variable "cluster_name" { - description = "What to name the Consul cluster and all of its associated resources" - type = string - default = "consul-example" -} - -variable "cluster_tag_key" { - description = "The tag the EC2 Instances will look for to automatically discover each other and form a cluster." - type = string - default = "consul-ci-load-test" -} diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 131d9057d2cc..c1b965235c9f 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -67,8 +67,7 @@ issue. Stale issues will be closed. ### Prerequisites If you wish to work on Consul itself, you'll first need to: -- install [Go](https://golang.org) (the version should match that of our - [CI config's](https://github.com/hashicorp/consul/blob/main/.circleci/config.yml) Go image). +- install [Go](https://golang.org) - [fork the Consul repo](../docs/contributing/fork-the-project.md) ### Building Consul diff --git a/.github/pr-labeler.yml b/.github/pr-labeler.yml index 1e371d5f9013..759130301bfb 100644 --- a/.github/pr-labeler.yml +++ b/.github/pr-labeler.yml @@ -57,7 +57,7 @@ theme/ui: # thinking: # type/bug: type/ci: - - .circleci/* + - .github/workflows/* # type/crash: type/docs: - website/**/* diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 2758e57c4aba..bcce35da60d6 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -79,7 +79,7 @@ jobs: matrix: partition: [1, 2, 3, 4] env: - EMBER_TEST_REPORT: test-results/report-oss.xml # outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml # outputs test report for CI test summary EMBER_TEST_PARALLEL: true # enables test parallelization with ember-exam CONSUL_NSPACES_ENABLED: ${{ endsWith(github.repository, '-enterprise') && 1 || 0 }} # NOTE: this should be 1 in ENT. JOBS: 2 # limit parallelism for broccoli-babel-transpiler diff --git a/.github/workflows/nightly-test-1.11.x.yaml b/.github/workflows/nightly-test-1.11.x.yaml index cd913d4eca49..2065738328ca 100644 --- a/.github/workflows/nightly-test-1.11.x.yaml +++ b/.github/workflows/nightly-test-1.11.x.yaml @@ -81,7 +81,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/nightly-test-1.12.x.yaml b/.github/workflows/nightly-test-1.12.x.yaml index 906f2ba8fb35..78d35746d693 100644 --- a/.github/workflows/nightly-test-1.12.x.yaml +++ b/.github/workflows/nightly-test-1.12.x.yaml @@ -81,7 +81,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/nightly-test-1.13.x.yaml b/.github/workflows/nightly-test-1.13.x.yaml index 2df70ff68807..0cd2870c78f9 100644 --- a/.github/workflows/nightly-test-1.13.x.yaml +++ b/.github/workflows/nightly-test-1.13.x.yaml @@ -81,7 +81,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/nightly-test-1.14.x.yaml b/.github/workflows/nightly-test-1.14.x.yaml index 745ad7608ee8..902ce25a7f60 100644 --- a/.github/workflows/nightly-test-1.14.x.yaml +++ b/.github/workflows/nightly-test-1.14.x.yaml @@ -81,7 +81,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/nightly-test-main.yaml b/.github/workflows/nightly-test-main.yaml index e823bac7b562..a645f8359c21 100644 --- a/.github/workflows/nightly-test-main.yaml +++ b/.github/workflows/nightly-test-main.yaml @@ -81,7 +81,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/reusable-lint.yml b/.github/workflows/reusable-lint.yml index 82650fd5e903..47b3846892af 100644 --- a/.github/workflows/reusable-lint.yml +++ b/.github/workflows/reusable-lint.yml @@ -49,7 +49,7 @@ jobs: uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # pin@v3.4.0 with: working-directory: ${{ matrix.directory }} - version: v1.51 + version: v1.51.1 args: --build-tags="${{ env.GOTAGS }}" -v - name: Notify Slack if: ${{ failure() }} diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 6ba474643382..dfbf02772f62 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -338,7 +338,7 @@ jobs: --latest-version latest ls -lrt env: - # this is needed because of incompatibility between RYUK container and circleci + # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 @@ -469,7 +469,7 @@ jobs: --latest-version "${{ env.CONSUL_VERSION }}" ls -lrt env: - # this is needed because of incompatibility between RYUK container and circleci + # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 diff --git a/GNUmakefile b/GNUmakefile index ee765693f4a9..02d1cbcc75fe 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -194,7 +194,7 @@ remote-docker: check-remote-dev-image-env --push \ -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ -# In CircleCI, the linux binary will be attached from a previous step at bin/. This make target +# In CI, the linux binary will be attached from a previous step at bin/. This make target # should only run in CI and not locally. ci.dev-docker: @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @@ -459,20 +459,10 @@ test-metrics-integ: test-compat-integ-setup --latest-version latest test-connect-ca-providers: -ifeq ("$(CIRCLECI)","true") -# Run in CI - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report.xml" -- -cover -coverprofile=coverage.txt ./agent/connect/ca -# Run leader tests that require Vault - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-leader.xml" -- -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul -# Run agent tests that require Vault - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-agent.xml" -- -cover -coverprofile=coverage-agent.txt -run Vault ./agent -else -# Run locally @echo "Running /agent/connect/ca tests in verbose mode" @go test -v ./agent/connect/ca @go test -v ./agent/consul -run Vault @go test -v ./agent -run Vault -endif .PHONY: proto proto: proto-tools proto-gen proto-mocks diff --git a/agent/checks/check_test.go b/agent/checks/check_test.go index 495fc1472bbc..a4bdf2ca3ed7 100644 --- a/agent/checks/check_test.go +++ b/agent/checks/check_test.go @@ -1147,9 +1147,6 @@ func TestCheckTCPPassing(t *testing.T) { if os.Getenv("TRAVIS") == "true" { t.Skip("IPV6 not supported on travis-ci") } - if os.Getenv("CIRCLECI") == "true" { - t.Skip("IPV6 not supported on CircleCI") - } tcpServer = mockTCPServer(`tcp6`) expectTCPStatus(t, tcpServer.Addr().String(), api.HealthPassing) tcpServer.Close() diff --git a/agent/connect/ca/testing.go b/agent/connect/ca/testing.go index 70c297732b86..450c13d6dd89 100644 --- a/agent/connect/ca/testing.go +++ b/agent/connect/ca/testing.go @@ -146,7 +146,7 @@ func NewTestVaultServer(t testing.T) *TestVaultServer { // We pass '-dev-no-store-token' to avoid having multiple vaults oddly // interact and fail like this: // - // Error initializing Dev mode: rename /home/circleci/.vault-token.tmp /home/circleci/.vault-token: no such file or directory + // Error initializing Dev mode: rename /.vault-token.tmp /.vault-token: no such file or directory // "-dev-no-store-token", } diff --git a/docs/README.md b/docs/README.md index 8f0be780011f..4a7523fd14bf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -48,7 +48,7 @@ contain other important source related to Consul. * [ui] contains the source code for the Consul UI. * [website] contains the source for [consul.io](https://www.consul.io/). A pull requests can update the source code and Consul's documentation at the same time. -* [.circleci] and [.github] contain the source for our CI and GitHub repository +* [.github] contains the source for our CI and GitHub repository automation. * [.changelog] contains markdown files that are used by [hashicorp/go-changelog] to produce the [CHANGELOG.md]. @@ -59,7 +59,6 @@ contain other important source related to Consul. [ui]: https://github.com/hashicorp/consul/tree/main/ui [website]: https://github.com/hashicorp/consul/tree/main/website -[.circleci]: https://github.com/hashicorp/consul/tree/main/.circleci [.github]: https://github.com/hashicorp/consul/tree/main/.github [.changelog]: https://github.com/hashicorp/consul/tree/main/.changelog [hashicorp/go-changelog]: https://github.com/hashicorp/go-changelog diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index 5afed0609037..e1cc5f8fe3fb 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -311,8 +311,7 @@ function pre_service_setup { } function start_services { - # Push the state to the shared docker volume (note this is because CircleCI - # can't use shared volumes) + # Push the state to the shared docker volume docker cp workdir/. envoy_workdir_1:/workdir # Start containers required @@ -473,8 +472,7 @@ function run_tests { # Wipe state wipe_volumes - # Push the state to the shared docker volume (note this is because CircleCI - # can't use shared volumes) + # Push the state to the shared docker volume docker cp workdir/. envoy_workdir_1:/workdir start_consul primary @@ -556,10 +554,6 @@ function suite_setup { echo "Rebuilding 'bats-verify' image..." retry_default docker build -t bats-verify -f Dockerfile-bats . - # if this fails on CircleCI your first thing to try would be to upgrade - # the machine image to the latest version using this listing: - # - # https://circleci.com/docs/2.0/configuration-reference/#available-linux-machine-images echo "Checking bats image..." docker run --rm -t bats-verify -v diff --git a/test/load/README.md b/test/load/README.md index f0b7f648b96e..23fd927a861b 100644 --- a/test/load/README.md +++ b/test/load/README.md @@ -4,9 +4,6 @@ Consul Load Testing is used to capture baseline performance metrics for Consul u This relies on the [Gruntwork's Terraform AWS Consul Module](https://github.com/hashicorp/terraform-aws-consul) which *by default* creates 3 Consul servers across 3 availability zones. A load test instance which has an image that is configured with the necessary scripts and [k6](https://k6.io/) is created and sends traffic to a load balancer. The load balancer will distribute requests across the Consul clients who will ultimately forward the requests to the servers. - -# Load Test Automation -This can only be run on PRs that a Dev Build has been made for. When a PR has the `pr/load-test` Github Label applied this will kick off a Github Action. This Github Action will trigger Circle CI to run a Terraform Apply that runs a load test against the Dev Build Consul binary. The GitHub Action will paste the CircleCI load test workflow URL to the PR as a comment. ## How to use [Terraform](https://www.terraform.io/downloads.html) and [Packer](https://www.packer.io/downloads), AWS and [Datadog](https://docs.datadoghq.com/getting_started/) are required to use this. All of this, except the AWS resources that will be utilized, are free. diff --git a/ui/packages/consul-ui/docs/upgrades.mdx b/ui/packages/consul-ui/docs/upgrades.mdx index 76cbab129794..7a830fc6ffbc 100644 --- a/ui/packages/consul-ui/docs/upgrades.mdx +++ b/ui/packages/consul-ui/docs/upgrades.mdx @@ -6,7 +6,7 @@ Node upgrades should be done when it is convienient, preferably using the latest Active LTS version (https://nodejs.org/en/about/releases/) that also corresponds with the ember version we are on (https://github.com/ember-cli/ember-cli/blob/master/docs/node-support.md) -Aswell as bumping our `.nvmrc` and `.circleci/config.yml` files, when bumping +Aswell as bumping our `.nvmrc` file, when bumping node versions, please check with the rest of the team to see if there are other repositories that should track the same node version as this repository. From a9edaf955ca4e3fd568fc1ea35cbf8fb2c348aaf Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 19 Apr 2023 13:51:48 -0400 Subject: [PATCH 174/421] backport of commit 7df53afce7e1d5e28fe296b15b760a4827e9c0ad (#17053) Co-authored-by: Kyle Havlovitz --- .changelog/17048.txt | 3 +++ lib/map_walker.go | 2 +- lib/map_walker_test.go | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .changelog/17048.txt diff --git a/.changelog/17048.txt b/.changelog/17048.txt new file mode 100644 index 000000000000..74f31c7ce275 --- /dev/null +++ b/.changelog/17048.txt @@ -0,0 +1,3 @@ +```release-note:bug +Fix an bug where decoding some Config structs with unset pointer fields could fail with `reflect: call of reflect.Value.Type on zero Value`. +``` diff --git a/lib/map_walker.go b/lib/map_walker.go index 8a2929098f10..b3779311be8b 100644 --- a/lib/map_walker.go +++ b/lib/map_walker.go @@ -110,7 +110,7 @@ func (w *mapWalker) MapElem(m, k, v reflect.Value) error { return nil } - if inner := v.Elem(); inner.Type() == typMapIfaceIface { + if inner := v.Elem(); inner.IsValid() && inner.Type() == typMapIfaceIface { // map[interface{}]interface{}, attempt to weakly decode into string keys var target map[string]interface{} if err := mapstructure.WeakDecode(v.Interface(), &target); err != nil { diff --git a/lib/map_walker_test.go b/lib/map_walker_test.go index 2642802f9ddb..068add7590a3 100644 --- a/lib/map_walker_test.go +++ b/lib/map_walker_test.go @@ -38,6 +38,16 @@ func TestMapWalk(t *testing.T) { }, unexpected: true, }, + // ensure we don't panic from trying to call reflect.Value.Type + // on a nil pointer + "nil pointer": { + input: map[string]interface{}{ + "foo": nil, + }, + expected: map[string]interface{}{ + "foo": nil, + }, + }, // ensure nested maps get processed correctly "nested": { input: map[string]interface{}{ From c67bbb36a397384544fc63a635779e6e35950780 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 19 Apr 2023 15:53:51 -0700 Subject: [PATCH 175/421] backport of commit d7993024329e32329903e58474dea451b2205ad0 (#17059) Co-authored-by: John Murret --- .github/workflows/test-integrations.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index dfbf02772f62..36f33c015bef 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -244,11 +244,6 @@ jobs: --jsonfile /tmp/jsonfile/go-test.log \ --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" - - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - with: - name: container-logs - path: ./test/integration/connect/envoy/workdir/logs - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: From 689360c0c5ae128cee7f8379b97dd9d569b86e10 Mon Sep 17 00:00:00 2001 From: Anita Akaeze Date: Thu, 20 Apr 2023 14:47:07 -0400 Subject: [PATCH 176/421] NET-3648: Add script to get consul and envoy version (#17060) (#17074) --- test/integration/consul-container/go.mod | 39 +++-- test/integration/consul-container/go.sum | 145 +++++++++++++++--- .../consul_envoy_version.go | 45 ++++++ 3 files changed, 191 insertions(+), 38 deletions(-) create mode 100644 test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index e13d6e4f1fd3..3eb170ccd5a7 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -7,6 +7,7 @@ require ( github.com/docker/docker v20.10.22+incompatible github.com/docker/go-connections v0.4.0 github.com/hashicorp/consul/api v1.20.0 + github.com/hashicorp/consul/envoyextensions v0.1.2 github.com/hashicorp/consul/sdk v0.13.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-multierror v1.1.1 @@ -24,17 +25,24 @@ require ( ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/Microsoft/hcsshim v0.9.4 // indirect - github.com/armon/go-metrics v0.3.10 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + fortio.org/dflag v1.5.2 // indirect + fortio.org/log v1.3.0 // indirect + fortio.org/sets v1.0.2 // indirect + fortio.org/version v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Microsoft/hcsshim v0.9.7 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b // indirect github.com/containerd/cgroups v1.0.4 // indirect - github.com/containerd/containerd v1.6.8 // indirect + github.com/containerd/containerd v1.6.19 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/envoyproxy/go-control-plane v0.10.3 // indirect + github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -50,8 +58,8 @@ require ( github.com/hashicorp/memberlist v0.5.0 // indirect github.com/itchyny/timefmt-go v0.1.4 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/miekg/dns v1.1.41 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -67,11 +75,12 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sirupsen/logrus v1.8.1 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.4.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737 // indirect - google.golang.org/grpc v1.49.0 // indirect + golang.org/x/exp v0.0.0-20230303215020-44a13b063f3e // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/tools v0.6.0 // indirect + google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -81,3 +90,5 @@ replace github.com/hashicorp/consul/api => ../../../api replace github.com/hashicorp/consul/sdk => ../../../sdk replace github.com/hashicorp/consul => ../../.. + +replace github.com/hashicorp/consul/envoyextensions => ../../../envoyextensions diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum index 010695dea029..60a8a25fec66 100644 --- a/test/integration/consul-container/go.sum +++ b/test/integration/consul-container/go.sum @@ -10,23 +10,33 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= @@ -48,8 +58,8 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= @@ -58,8 +68,8 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= -github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.7 h1:mKNHW/Xvv1aFH87Jb6ERDzXTJTLPlmzfZ28VBFD/bfg= +github.com/Microsoft/hcsshim v0.9.7/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -78,8 +88,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -102,9 +112,10 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8n github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -126,7 +137,14 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b h1:ACGZRIr7HsgBKHsueQ1yM4WaVaXh21ynwqsF8M8tXhA= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -164,8 +182,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= -github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= +github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= +github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -287,12 +305,19 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.3 h1:xdCVXxEe0Y3FQith+0cj2irwZudqGYvecuLB1HtdexY= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -341,6 +366,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -354,6 +380,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -370,8 +397,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -379,6 +407,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -386,17 +415,20 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -421,6 +453,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= @@ -466,6 +499,7 @@ github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -499,6 +533,7 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -509,6 +544,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -521,16 +557,18 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -644,6 +682,7 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -710,6 +749,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -735,6 +776,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -769,7 +811,9 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= @@ -785,9 +829,11 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -800,6 +846,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -828,6 +875,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -837,8 +885,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -870,6 +919,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -879,6 +932,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= @@ -887,6 +941,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -898,8 +953,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -950,10 +1004,15 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -974,6 +1033,7 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -992,8 +1052,11 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1039,14 +1102,26 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1061,8 +1136,13 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1089,14 +1169,25 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737 h1:K1zaaMdYBXRyX+cwFnxj7M6zwDyumLQMZ5xqwGvjreQ= -google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737/go.mod h1:2r/26NEF3bFmT3eC3aZreahSal0C3Shl8Gi6vyDYqOQ= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1109,14 +1200,18 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1130,6 +1225,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -1174,6 +1270,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= diff --git a/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go b/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go new file mode 100644 index 000000000000..3f56fa4ea628 --- /dev/null +++ b/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go @@ -0,0 +1,45 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + "github.com/hashicorp/consul/envoyextensions/xdscommon" +) + +type consulEnvoyVersions struct { + ConsulVersion string + EnvoyVersions []string +} + +func main() { + cev := consulEnvoyVersions{} + + // Get Consul Version + data, err := os.ReadFile("./version/VERSION") + if err != nil { + panic(err) + } + cVersion := strings.TrimSpace(string(data)) + + cev.EnvoyVersions = append(cev.EnvoyVersions, xdscommon.EnvoyVersions...) + + // ensure the versions are properly sorted latest to oldest + sort.Sort(sort.Reverse(sort.StringSlice(cev.EnvoyVersions))) + + ceVersions := consulEnvoyVersions{ + ConsulVersion: string(cVersion), + EnvoyVersions: cev.EnvoyVersions, + } + output, err := json.Marshal(ceVersions) + if err != nil { + panic(err) + } + fmt.Print(string(output)) +} From eaf0c05217a01dd24a092544264e70c70c143e1d Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 20 Apr 2023 15:02:55 -0700 Subject: [PATCH 177/421] backport of commit f42b1de80b88aedaaf8271368c377701ed80ad1e (#17079) Co-authored-by: R.B. Boyer --- .../test/consul_envoy_version/consul_envoy_version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go b/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go index 3f56fa4ea628..72c4964ffbf4 100644 --- a/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go +++ b/test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go @@ -34,7 +34,7 @@ func main() { sort.Sort(sort.Reverse(sort.StringSlice(cev.EnvoyVersions))) ceVersions := consulEnvoyVersions{ - ConsulVersion: string(cVersion), + ConsulVersion: cVersion, EnvoyVersions: cev.EnvoyVersions, } output, err := json.Marshal(ceVersions) From fa621379e2099da8483b1d29677e13a2f9f04e81 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Thu, 20 Apr 2023 20:33:47 -0700 Subject: [PATCH 178/421] backport of commit 6b03c7b93ffb87623bd318c52929e3f16fab9101 (#16936) Co-authored-by: Andrea Scarpino --- .../docs/connect/config-entries/service-defaults.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 0f9f7cfa9789..35cd72bf31fc 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -66,7 +66,7 @@ The following outline shows how to format the service splitter configuration ent - [`Port`](#destination): integer | `0` - [`MaxInboundConnections`](#maxinboundconnections): integer | `0` - [`LocalConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` -- [`LocalRequestTiimeoutMs`](#localrequesttimeoutms): integer | `0` +- [`LocalRequestTimeoutMs`](#localrequesttimeoutms): integer | `0` - [`MeshGateway`](#meshgateway): map | no default - [`Mode`](#meshgateway): string | no default - [`ExternalSNI`](#externalsni): string | no default @@ -133,7 +133,7 @@ The following outline shows how to format the service splitter configuration ent - [`port`](#destination): integer | `0` - [`maxInboundConnections`](#maxinboundconnections): integer | `0` - [`localConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` - - [`localRequestTiimeoutMs`](#localrequesttimeoutms): integer | `0` + - [`localRequestTimeoutMs`](#localrequesttimeoutms): integer | `0` - [`meshGateway`](#meshgateway): map | no default - [`mode`](#meshgateway): string | no default - [`externalSNI`](#externalsni): string | no default @@ -2033,4 +2033,4 @@ Reading a `service-defaults` config entry requires `service:read` on the resourc Creating, updating, or deleting a `service-defaults` config entry requires `service:write` on the resource. ---> \ No newline at end of file +--> From fc4d18deadd0d09b8521c3fef2f1f7c8f12b77ef Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 21 Apr 2023 08:16:53 -0700 Subject: [PATCH 179/421] feat: set up reporting agent (#16991) (#17031) Co-authored-by: Poonam Jadhav --- agent/consul/acl_server.go | 2 +- agent/consul/leader.go | 8 +++++- agent/consul/leader_connect.go | 8 +++--- agent/consul/leader_intentions.go | 4 +-- agent/consul/leader_intentions_test.go | 2 +- agent/consul/leader_test.go | 6 ++-- agent/consul/reporting/reporting.go | 38 +++++++++++++++++++++++++ agent/consul/reporting/reporting_oss.go | 21 ++++++++++++++ agent/consul/server.go | 6 ++++ agent/consul/server_oss.go | 6 ++++ agent/consul/system_metadata.go | 4 +-- agent/consul/system_metadata_test.go | 8 +++--- 12 files changed, 95 insertions(+), 18 deletions(-) create mode 100644 agent/consul/reporting/reporting.go create mode 100644 agent/consul/reporting/reporting_oss.go diff --git a/agent/consul/acl_server.go b/agent/consul/acl_server.go index 48e80191e384..5ea8f915cfd2 100644 --- a/agent/consul/acl_server.go +++ b/agent/consul/acl_server.go @@ -110,7 +110,7 @@ type serverACLResolverBackend struct { } func (s *serverACLResolverBackend) IsServerManagementToken(token string) bool { - mgmt, err := s.getSystemMetadata(structs.ServerManagementTokenAccessorID) + mgmt, err := s.GetSystemMetadata(structs.ServerManagementTokenAccessorID) if err != nil { s.logger.Debug("failed to fetch server management token: %w", err) return false diff --git a/agent/consul/leader.go b/agent/consul/leader.go index d5eb00fbbfec..76c3380700eb 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -338,6 +338,10 @@ func (s *Server) establishLeadership(ctx context.Context) error { s.startLogVerification(ctx) } + if s.config.Reporting.License.Enabled && s.reportingManager != nil { + s.reportingManager.StartReportingAgent() + } + s.logger.Debug("successfully established leadership", "duration", time.Since(start)) return nil } @@ -374,6 +378,8 @@ func (s *Server) revokeLeadership() { s.resetConsistentReadReady() s.autopilot.DisableReconciliation() + + s.reportingManager.StopReportingAgent() } // initializeACLs is used to setup the ACLs if we are the leader @@ -522,7 +528,7 @@ func (s *Server) initializeACLs(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to generate the secret ID for the server management token: %w", err) } - if err := s.setSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil { + if err := s.SetSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil { return fmt.Errorf("failed to persist server management token: %w", err) } diff --git a/agent/consul/leader_connect.go b/agent/consul/leader_connect.go index 0c57bc13a7f8..7166ad75cd61 100644 --- a/agent/consul/leader_connect.go +++ b/agent/consul/leader_connect.go @@ -204,7 +204,7 @@ func (s *Server) setVirtualIPFlags() (bool, error) { } func (s *Server) setVirtualIPVersionFlag() (bool, error) { - val, err := s.getSystemMetadata(structs.SystemMetadataVirtualIPsEnabled) + val, err := s.GetSystemMetadata(structs.SystemMetadataVirtualIPsEnabled) if err != nil { return false, err } @@ -217,7 +217,7 @@ func (s *Server) setVirtualIPVersionFlag() (bool, error) { minVirtualIPVersion.String()) } - if err := s.setSystemMetadataKey(structs.SystemMetadataVirtualIPsEnabled, "true"); err != nil { + if err := s.SetSystemMetadataKey(structs.SystemMetadataVirtualIPsEnabled, "true"); err != nil { return false, nil } @@ -225,7 +225,7 @@ func (s *Server) setVirtualIPVersionFlag() (bool, error) { } func (s *Server) setVirtualIPTerminatingGatewayVersionFlag() (bool, error) { - val, err := s.getSystemMetadata(structs.SystemMetadataTermGatewayVirtualIPsEnabled) + val, err := s.GetSystemMetadata(structs.SystemMetadataTermGatewayVirtualIPsEnabled) if err != nil { return false, err } @@ -238,7 +238,7 @@ func (s *Server) setVirtualIPTerminatingGatewayVersionFlag() (bool, error) { minVirtualIPTerminatingGatewayVersion.String()) } - if err := s.setSystemMetadataKey(structs.SystemMetadataTermGatewayVirtualIPsEnabled, "true"); err != nil { + if err := s.SetSystemMetadataKey(structs.SystemMetadataTermGatewayVirtualIPsEnabled, "true"); err != nil { return false, nil } diff --git a/agent/consul/leader_intentions.go b/agent/consul/leader_intentions.go index 9adc26795deb..2894acd58460 100644 --- a/agent/consul/leader_intentions.go +++ b/agent/consul/leader_intentions.go @@ -22,7 +22,7 @@ func (s *Server) startIntentionConfigEntryMigration(ctx context.Context) error { // Check for the system metadata first, as that's the most trustworthy in // both the primary and secondaries. - intentionFormat, err := s.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) if err != nil { return err } @@ -240,7 +240,7 @@ func (s *Server) legacyIntentionMigrationInSecondaryDC(ctx context.Context) erro // error. for { // Check for the system metadata first, as that's the most trustworthy. - intentionFormat, err := s.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) if err != nil { return err } diff --git a/agent/consul/leader_intentions_test.go b/agent/consul/leader_intentions_test.go index e0dcb8b3d10b..89dbc9d6256c 100644 --- a/agent/consul/leader_intentions_test.go +++ b/agent/consul/leader_intentions_test.go @@ -520,7 +520,7 @@ func TestLeader_LegacyIntentionMigration(t *testing.T) { // Wait until the migration routine is complete. retry.Run(t, func(r *retry.R) { - intentionFormat, err := s1.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s1.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) require.NoError(r, err) if intentionFormat != structs.SystemMetadataIntentionFormatConfigValue { r.Fatal("intention migration is not yet complete") diff --git a/agent/consul/leader_test.go b/agent/consul/leader_test.go index e8bcb39a6533..fe8e1b678a71 100644 --- a/agent/consul/leader_test.go +++ b/agent/consul/leader_test.go @@ -1300,7 +1300,7 @@ func TestLeader_ACL_Initialization(t *testing.T) { require.NoError(t, err) require.NotNil(t, policy) - serverToken, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken, err := s1.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken) @@ -1338,14 +1338,14 @@ func TestLeader_ACL_Initialization_SecondaryDC(t *testing.T) { testrpc.WaitForTestAgent(t, s2.RPC, "dc2") // Check dc1's management token - serverToken1, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken1, err := s1.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken1) _, err = uuid.ParseUUID(serverToken1) require.NoError(t, err) // Check dc2's management token - serverToken2, err := s2.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken2, err := s2.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken2) _, err = uuid.ParseUUID(serverToken2) diff --git a/agent/consul/reporting/reporting.go b/agent/consul/reporting/reporting.go new file mode 100644 index 000000000000..982ac0225d58 --- /dev/null +++ b/agent/consul/reporting/reporting.go @@ -0,0 +1,38 @@ +package reporting + +import ( + "sync" + + "github.com/hashicorp/go-hclog" +) + +type ReportingManager struct { + logger hclog.Logger + server ServerDelegate + EntDeps + sync.RWMutex +} + +const ( + SystemMetadataReportingProcessID = "reporting-process-id" +) + +//go:generate mockery --name ServerDelegate --inpackage +type ServerDelegate interface { + GetSystemMetadata(key string) (string, error) + SetSystemMetadataKey(key, val string) error + IsLeader() bool +} + +func NewReportingManager(logger hclog.Logger, deps EntDeps, server ServerDelegate) *ReportingManager { + rm := &ReportingManager{ + logger: logger.Named("reporting"), + server: server, + } + err := rm.initEnterpriseReporting(deps) + if err != nil { + rm.logger.Error("Error initializing reporting manager", "error", err) + return nil + } + return rm +} diff --git a/agent/consul/reporting/reporting_oss.go b/agent/consul/reporting/reporting_oss.go new file mode 100644 index 000000000000..673559902506 --- /dev/null +++ b/agent/consul/reporting/reporting_oss.go @@ -0,0 +1,21 @@ +//go:build !consulent +// +build !consulent + +package reporting + +type EntDeps struct{} + +func (rm *ReportingManager) initEnterpriseReporting(entDeps EntDeps) error { + // no op + return nil +} + +func (rm *ReportingManager) StartReportingAgent() error { + // no op + return nil +} + +func (rm *ReportingManager) StopReportingAgent() error { + // no op + return nil +} diff --git a/agent/consul/server.go b/agent/consul/server.go index d2c8c3cfe332..b3187ba1b0e5 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -39,6 +39,7 @@ import ( "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/multilimiter" rpcRate "github.com/hashicorp/consul/agent/consul/rate" + "github.com/hashicorp/consul/agent/consul/reporting" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/stream" "github.com/hashicorp/consul/agent/consul/usagemetrics" @@ -409,6 +410,9 @@ type Server struct { // embedded struct to hold all the enterprise specific data EnterpriseServer operatorServer *operator.Server + + // handles metrics reporting to HashiCorp + reportingManager *reporting.ReportingManager } type connHandler interface { @@ -728,6 +732,8 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom s.overviewManager = NewOverviewManager(s.logger, s.fsm, s.config.MetricsReportingInterval) go s.overviewManager.Run(&lib.StopChannelContext{StopCh: s.shutdownCh}) + s.reportingManager = reporting.NewReportingManager(s.logger, getEnterpriseReportingDeps(flat), s) + // Initialize external gRPC server - register services on external gRPC server. s.externalACLServer = aclgrpc.NewServer(aclgrpc.Config{ ACLsEnabled: s.config.ACLsEnabled, diff --git a/agent/consul/server_oss.go b/agent/consul/server_oss.go index 67731e450195..5e285c96353e 100644 --- a/agent/consul/server_oss.go +++ b/agent/consul/server_oss.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc" "github.com/hashicorp/consul/acl" + "github.com/hashicorp/consul/agent/consul/reporting" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/lib" ) @@ -178,3 +179,8 @@ func addSerfMetricsLabels(conf *serf.Config, wan bool, segment string, partition func (s *Server) updateReportingConfig(config ReloadableConfig) { // no-op } + +func getEnterpriseReportingDeps(deps Deps) reporting.EntDeps { + // no-op + return reporting.EntDeps{} +} diff --git a/agent/consul/system_metadata.go b/agent/consul/system_metadata.go index f1603225738f..5ceb5fc5e717 100644 --- a/agent/consul/system_metadata.go +++ b/agent/consul/system_metadata.go @@ -4,7 +4,7 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -func (s *Server) getSystemMetadata(key string) (string, error) { +func (s *Server) GetSystemMetadata(key string) (string, error) { _, entry, err := s.fsm.State().SystemMetadataGet(nil, key) if err != nil { return "", err @@ -16,7 +16,7 @@ func (s *Server) getSystemMetadata(key string) (string, error) { return entry.Value, nil } -func (s *Server) setSystemMetadataKey(key, val string) error { +func (s *Server) SetSystemMetadataKey(key, val string) error { args := &structs.SystemMetadataRequest{ Op: structs.SystemMetadataUpsert, Entry: &structs.SystemMetadataEntry{Key: key, Value: val}, diff --git a/agent/consul/system_metadata_test.go b/agent/consul/system_metadata_test.go index 633aa6bc4078..b17790d22469 100644 --- a/agent/consul/system_metadata_test.go +++ b/agent/consul/system_metadata_test.go @@ -39,9 +39,9 @@ func TestLeader_SystemMetadata_CRUD(t *testing.T) { require.Len(t, entries, 0) // Create 3 - require.NoError(t, srv.setSystemMetadataKey("key1", "val1")) - require.NoError(t, srv.setSystemMetadataKey("key2", "val2")) - require.NoError(t, srv.setSystemMetadataKey("key3", "")) + require.NoError(t, srv.SetSystemMetadataKey("key1", "val1")) + require.NoError(t, srv.SetSystemMetadataKey("key2", "val2")) + require.NoError(t, srv.SetSystemMetadataKey("key3", "")) mapify := func(entries []*structs.SystemMetadataEntry) map[string]string { m := make(map[string]string) @@ -62,7 +62,7 @@ func TestLeader_SystemMetadata_CRUD(t *testing.T) { }, mapify(entries)) // Update one and delete one. - require.NoError(t, srv.setSystemMetadataKey("key3", "val3")) + require.NoError(t, srv.SetSystemMetadataKey("key3", "val3")) require.NoError(t, srv.deleteSystemMetadataKey("key1")) _, entries, err = state.SystemMetadataList(nil) From 5493cf6af5e999db98a191a36d748510038a4788 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 21 Apr 2023 09:58:49 -0700 Subject: [PATCH 180/421] Backport of ci: fix test splits that have less test packages than runner count from hanging into release/1.15.x (#17085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cli: remove stray whitespace when loading the consul version from the VERSION file (#16467) Fixes a regression from #15631 in the output of `consul version` from: Consul v1.16.0-dev +ent Revision 56b86acbe5+CHANGES to Consul v1.16.0-dev+ent Revision 56b86acbe5+CHANGES * Docs/services refactor docs day 122022 (#16103) * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * converted main services page to services overview page * set up services usage dirs * added Define Services usage page * converted health checks everything page to Define Health Checks usage page * added Register Services and Nodes usage page * converted Query with DNS to Discover Services and Nodes Overview page * added Configure DNS Behavior usage page * added Enable Static DNS Lookups usage page * added the Enable Dynamic Queries DNS Queries usage page * added the Configuration dir and overview page - may not need the overview, tho * fixed the nav from previous commit * added the Services Configuration Reference page * added Health Checks Configuration Reference page * updated service defaults configuraiton entry to new configuration ref format * fixed some bad links found by checker * more bad links found by checker * another bad link found by checker * fixed cross-links between new topics * updated links to the new services pages * fixed bad links in scale file * tweaks to titles and phrasing * fixed typo in checks.mdx * started updating the conf ref to latest template * update SD conf ref to match latest CT standard * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> * remove previous version of the checks page * fixed cross-links * Apply suggestions from code review Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> --------- Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> * docs: clarify license expiration upgrade behavior (#16464) * add provider ca auth-method support for azure Does the required dance with the local HTTP endpoint to get the required data for the jwt based auth setup in Azure. Keeps support for 'legacy' mode where all login data is passed on via the auth methods parameters. Refactored check for hardcoded /login fields. * Changed titles for services pages to sentence style cap (#16477) * Changed titles for services pages to sentence style cap * missed a meta title * docs: Consul 1.15.0 and Consul K8s 1.0 release notes (#16481) * add new release notes --------- Co-authored-by: Tu Nguyen * fix (cli): return error msg if acl policy not found (#16485) * fix: return error msg if acl policy not found * changelog * add test * update services nav titles (#16484) * Improve ux to help users avoid overwriting fields of ACL tokens, roles and policies (#16288) * Deprecate merge-policies and add options add-policy-name/add-policy-id to improve CLI token update command * deprecate merge-roles fields * Fix potential flakey tests and update ux to remove 'completely' + typo fixes * NET-2292: port ingress-gateway test case "http" from BATS addendum (#16490) * docs: Update release notes with Envoy compat issue (#16494) * Update v1_15_x.mdx --------- Co-authored-by: Tu Nguyen * Suppress AlreadyRegisteredError to fix test retries (#16501) * Suppress AlreadyRegisteredError to fix test retries * Remove duplicate sink * Speed up test by registering services concurrently (#16509) * add provider ca support for jwt file base auth Adds support for a jwt token in a file. Simply reads the file and sends the read in jwt along to the vault login. It also supports a legacy mode with the jwt string being passed directly. In which case the path is made optional. * docs(architecture): remove merge conflict leftovers (#16507) * add provider ca auth support for kubernetes Adds support for Kubernetes jwt/token file based auth. Only needs to read the file and save the contents as the jwt/token. * Merge pull request #4538 from hashicorp/NET-2396 (#16516) NET-2396: refactor test to reduce duplication * Merge pull request #4584 from hashicorp/refactor_cluster_config (#16517) NET-2841: PART 1 - refactor NewPeeringCluster to support custom config * Add ServiceResolver RequestTimeout for route timeouts to make TerminatingGateway upstream timeouts configurable (#16495) * Leverage ServiceResolver ConnectTimeout for route timeouts to make TerminatingGateway upstream timeouts configurable * Regenerate golden files * Add RequestTimeout field * Add changelog entry * Fix issue where terminating gateway service resolvers weren't properly cleaned up (#16498) * Fix issue where terminating gateway service resolvers weren't properly cleaned up * Add integration test for cleaning up resolvers * Add changelog entry * Use state test and drop integration test * Add support for failover policies (#16505) * modified unsupported envoy version error (#16518) - When an envoy version is out of a supported range, we now return the envoy version being used as `major.minor.x` to indicate that it is the minor version at most that is incompatible - When an envoy version is in the list of unsupported envoy versions we return back the envoy version in the error message as `major.minor.patch` as now the exact version matters. * Remove private prefix from proto-gen-rpc-glue e2e test (#16433) * Fix resolution of service resolvers with subsets for external upstreams (#16499) * Fix resolution of service resolvers with subsets for external upstreams * Add tests * Add changelog entry * Update view filter logic * fixed broken links associated with cluster peering updates (#16523) * fixed broken links associated with cluster peering updates * additional links to fix * typos * fixed redirect file * add provider ca support for approle auth-method Adds support for the approle auth-method. Only handles using the approle role/secret to auth and it doesn't support the agent's extra management configuration options (wrap and delete after read) as they are not required as part of the auth (ie. they are vault agent things). * update connect/ca's vault AuthMethod conf section (#16346) Updated Params field to re-frame as supporting arguments specific to the supported vault-agent auth-auth methods with links to each methods "#configuration" section. Included a call out limits on parameters supported. * proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher (#16497) Receiving an "acl not found" error from an RPC in the agent cache and the streaming/event components will cause any request loops to cease under the assumption that they will never work again if the token was destroyed. This prevents log spam (#14144, #9738). Unfortunately due to things like: - authz requests going to stale servers that may not have witnessed the token creation yet - authz requests in a secondary datacenter happening before the tokens get replicated to that datacenter - authz requests from a primary TO a secondary datacenter happening before the tokens get replicated to that datacenter The caller will get an "acl not found" *before* the token exists, rather than just after. The machinery added above in the linked PRs will kick in and prevent the request loop from looping around again once the tokens actually exist. For `consul-dataplane` usages, where xDS is served by the Consul servers rather than the clients ultimately this is not a problem because in that scenario the `agent/proxycfg` machinery is on-demand and launched by a new xDS stream needing data for a specific service in the catalog. If the watching goroutines are terminated it ripples down and terminates the xDS stream, which CDP will eventually re-establish and restart everything. For Consul client usages, the `agent/proxycfg` machinery is ahead-of-time launched at service registration time (called "local" in some of the proxycfg machinery) so when the xDS stream comes in the data is already ready to go. If the watching goroutines terminate it should terminate the xDS stream, but there's no mechanism to re-spawn the watching goroutines. If the xDS stream reconnects it will see no `ConfigSnapshot` and will not get one again until the client agent is restarted, or the service is re-registered with something changed in it. This PR fixes a few things in the machinery: - there was an inadvertent deadlock in fetching snapshot from the proxycfg machinery by xDS, such that when the watching goroutine terminated the snapshots would never be fetched. This caused some of the xDS machinery to get indefinitely paused and not finish the teardown properly. - Every 30s we now attempt to re-insert all locally registered services into the proxycfg machinery. - When services are re-inserted into the proxycfg machinery we special case "dead" ones such that we unilaterally replace them rather that doing that conditionally. * NET-2903 Normalize weight for http routes (#16512) * NET-2903 Normalize weight for http routes * Update website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Add some basic UI improvements for api-gateway services (#16508) * Add some basic ui improvements for api-gateway services * Add changelog entry * Use ternary for null check * Update gateway doc links * rename changelog entry for new PR * Fix test * fixes empty link in DNS usage page (#16534) * NET-2904 Fixes API Gateway Route Service Weight Division Error * Improve ux around ACL token to help users avoid overwriting node/service identities (#16506) * Deprecate merge-node-identities and merge-service-identities flags * added tests for node identities changes * added changelog file and docs * Follow-up fixes to consul connect envoy command (#16530) * Merge pull request #4573 from hashicorp/NET-2841 (#16544) * Merge pull request #4573 from hashicorp/NET-2841 NET-2841: PART 2 refactor upgrade tests to include version 1.15 * update upgrade versions * upgrade test: discovery chain across partition (#16543) * Update the consul-k8s cli docs for the new `proxy log` subcommand (#16458) * Update the consul-k8s cli docs for the new `proxy log` subcommand * Updated consul-k8s docs from PR feedback * Added proxy log command to release notes * Delete test-link-rewrites.yml (#16546) * feat: update notification to use hds toast component (#16519) * Fix flakey tests related to ACL token updates (#16545) * Fix flakey tests related to ACL token updates * update all acl token update tests * extra create_token function to its own thing * support vault auth config for alicloud ca provider Add support for using existing vault auto-auth configurations as the provider configuration when using Vault's CA provider with AliCloud. AliCloud requires 2 extra fields to enable it to use STS (it's preferred auth setup). Our vault-plugin-auth-alicloud package contained a method to help generate them as they require you to make an http call to a faked endpoint proxy to get them (url and headers base64 encoded). * Update docs to reflect functionality (#16549) * Update docs to reflect functionality * make consistent with other client runtimes * upgrade test: use retry with ModifyIndex and remove ent test file (#16553) * add agent locality and replicate it across peer streams (#16522) * docs: Document config entry permissions (#16556) * Broken link fixes (#16566) * NET-2954: Improve integration tests CI execution time (#16565) * NET-2954: Improve integration tests CI execution time * fix ci * remove comments and modify config file * fix bug that can lead to peering service deletes impacting the state of local services (#16570) * Update changelog with patch releases (#16576) * Bump submodules from latest 1.15.1 patch release (#16578) * Update changelog with Consul patch releases 1.13.7, 1.14.5, 1.15.1 * Bump submodules from latest patch release * Forgot one * website: adds content-check command and README update (#16579) * added a backport-checker GitHub action (#16567) * added a backport-checker GitHub action * Update .github/workflows/backport-checker.yml * auto-updated agent/uiserver/dist/ from commit 63204b518 (#16587) Co-authored-by: hc-github-team-consul-core * GRPC stub for the ResourceService (#16528) * UI: Fix htmlsafe errors throughout the app (#16574) * Upgrade ember-intl * Add changelog * Add yarn lock * Add namespace file with build tag for OSS gateway tests (#16590) * Add namespace file with build tag for OSS tests * Remove TODO comment * JIRA pr check: Filter out OSS/ENT merges (#16593) * jira pr check filter out dependabot and oss/ent merges * allow setting locality on services and nodes (#16581) * Add Peer Locality to Discovery Chains (#16588) Add peer locality to discovery chains * fixes for unsupported partitions field in CRD metadata block (#16604) * fixes for unsupported partitions field in CRD metadata block * Apply suggestions from code review Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --------- Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> * Create a weekly 404 checker for all Consul docs content (#16603) * Consul WAN Fed with Vault Secrets Backend document updates (#16597) * Consul WAN Fed with Vault Secrets Backend document updates * Corrected dc1-consul.yaml and dc2-consul.yaml file highlights * Update website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Allow HCP metrics collection for Envoy proxies Co-authored-by: Ashvitha Sridharan Co-authored-by: Freddy Add a new envoy flag: "envoy_hcp_metrics_bind_socket_dir", a directory where a unix socket will be created with the name `_.sock` to forward Envoy metrics. If set, this will configure: - In bootstrap configuration a local stats_sink and static cluster. These will forward metrics to a loopback listener sent over xDS. - A dynamic listener listening at the socket path that the previously defined static cluster is sending metrics to. - A dynamic cluster that will forward traffic received at this listener to the hcp-metrics-collector service. Reasons for having a static cluster pointing at a dynamic listener: - We want to secure the metrics stream using TLS, but the stats sink can only be defined in bootstrap config. With dynamic listeners/clusters we can use the proxy's leaf certificate issued by the Connect CA, which isn't available at bootstrap time. - We want to intelligently route to the HCP collector. Configuring its addreess at bootstrap time limits our flexibility routing-wise. More on this below. Reasons for defining the collector as an upstream in `proxycfg`: - The HCP collector will be deployed as a mesh service. - Certificate management is taken care of, as mentioned above. - Service discovery and routing logic is automatically taken care of, meaning that no code changes are required in the xds package. - Custom routing rules can be added for the collector using discovery chain config entries. Initially the collector is expected to be deployed to each admin partition, but in the future could be deployed centrally in the default partition. These config entries could even be managed by HCP itself. * Add copywrite setup file (#16602) * Add sameness-group configuration entry. (#16608) This commit adds a sameness-group config entry to the API and structs packages. It includes some validation logic and a new memdb index that tracks the default sameness-group for each partition. Sameness groups will simplify the effort of managing failovers / intentions / exports for peers and partitions. Note that this change purely to introduce the configuration entry and does not include the full functionality of sameness-groups. * Preserve CARoots when updating Vault CA configuration (#16592) If a CA config update did not cause a root change, the codepath would return early and skip some steps which preserve its intermediate certificates and signing key ID. This commit re-orders some code and prevents updates from generating new intermediate certificates. * Add UI copyright headers files (#16614) * Add copyright headers to UI files * Ensure copywrite file ignores external libs * Docs discovery typo (#16628) * docs(discovery): typo * docs(discovery): EOF and trim lines --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Fix issue with trust bundle read ACL check. (#16630) This commit fixes an issue where trust bundles could not be read by services in a non-default namespace, unless they had excessive ACL permissions given to them. Prior to this change, `service:write` was required in the default namespace in order to read the trust bundle. Now, `service:write` to a service in any namespace is sufficient. * Basic resource type registry (#16622) * Backport ENT-4704 (#16612) * feat: update typography to consume hds styles (#16577) * Add known issues to Raft WAL docs. (#16600) * Add known issues to Raft WAL docs. * Refactor update based on review feedback * Tune 404 checker to exclude false-positives and use intended file path (#16636) * Update e2e tests for namespaces (#16627) * Refactored "NewGatewayService" to handle namespaces, fixed TestHTTPRouteFlattening test * Fixed existing http_route tests for namespacing * Squash aclEnterpriseMeta for ResourceRefs and HTTPServices, accept namespace for creating connect services and regular services * Use require instead of assert after creating namespaces in http_route_tests * Refactor NewConnectService and NewGatewayService functions to use cfg objects to reduce number of method args * Rename field on SidecarConfig in tests from `SidecarServiceName` to `Name` to avoid stutter * net 2731 ip config entry OSS version (#16642) * ip config entry * name changing * move to ent * ent version * renaming * change format * renaming * refactor * add default values * fix confusing spiffe ids in golden tests (#16643) * First cluster grpc service should be NodePort for the second cluster to connect (#16430) * First cluster grpc service should be NodePort This is based on the issue opened here https://github.com/hashicorp/consul-k8s/issues/1903 If you follow the documentation https://developer.hashicorp.com/consul/docs/k8s/deployment-configurations/single-dc-multi-k8s exactly as it is, the first cluster will only create the consul UI service on NodePort but not the rest of the services (including for grpc). By default, from the helm chart, they are created as headless services by setting clusterIP None. This will cause an issue for the second cluster to discover consul server on the first cluster over gRPC as it cannot simply cannot through gRPC default port 8502 and it ends up in an error as shown in the issue https://github.com/hashicorp/consul-k8s/issues/1903 As a solution, the grpc service should be exposed using NodePort (or LoadBalancer). I added those changes required in both cluster1-values.yaml and cluster2-values.yaml, and also a description for those changes for the normal users to understand. Kindly review and I hope this PR will be accepted. * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Update website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --------- Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * Add in query options for catalog service existing in a specific (#16652) namespace when creating service for tests * fix: add AccessorID property to PUT token request (#16660) * add sameness group support to service resolver failover and redirects (#16664) * Fix incorrect links on Envoy extensions documentation (#16666) * [API Gateway] Fix invalid cluster causing gateway programming delay (#16661) * Add test for http routes * Add fix * Fix tests * Add changelog entry * Refactor and fix flaky tests * Bump tomhjp/gh-action-jira-search from 0.2.1 to 0.2.2 (#16667) Bumps [tomhjp/gh-action-jira-search](https://github.com/tomhjp/gh-action-jira-search) from 0.2.1 to 0.2.2. - [Release notes](https://github.com/tomhjp/gh-action-jira-search/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-search/compare/v0.2.1...v0.2.2) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-search dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump atlassian/gajira-transition from 2.0.1 to 3.0.1 (#15921) Bumps [atlassian/gajira-transition](https://github.com/atlassian/gajira-transition) from 2.0.1 to 3.0.1. - [Release notes](https://github.com/atlassian/gajira-transition/releases) - [Commits](https://github.com/atlassian/gajira-transition/compare/v2.0.1...v3.0.1) --- updated-dependencies: - dependency-name: atlassian/gajira-transition dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * Snapshot restore tests (#16647) * add snapshot restore test * add logstore as test parameter * Use the correct image version * make sure we read the logs from a followers to test the follower snapshot install path. * update to raf-wal v0.3.0 * add changelog. * updating changelog for bug description and removed integration test. * setting up test container builder to only set logStore for 1.15 and higher --------- Co-authored-by: Paul Banks Co-authored-by: John Murret * add sameness groups to discovery chains (#16671) * feat: add category annotation to RPC and gRPC methods (#16646) * Update GH actions to create Jira issue automatically (#16656) * Adds check to verify that the API Gateway is being created with at least one listener * Fix route subscription when using namespaces (#16677) * Fix route subscription when using namespaces * Update changelog * Fix changelog entry to reference that the bug was enterprise only * peering: peering partition failover fixes (#16673) add local source partition for peered upstreams * fix jira sync actions, remove custom fields (#16686) * Docs/update jira sync pr issue (#16688) * fix jira sync actions, remove custom fields * remove more additional fields, debug * Docs: Jira sync Update issuetype to bug (#16689) * update issuetype to bug * fix conditional for pr edu * build(deps): bump tomhjp/gh-action-jira-create from 0.2.0 to 0.2.1 (#16685) Bumps [tomhjp/gh-action-jira-create](https://github.com/tomhjp/gh-action-jira-create) from 0.2.0 to 0.2.1. - [Release notes](https://github.com/tomhjp/gh-action-jira-create/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-create/compare/v0.2.0...v0.2.1) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-create dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * build(deps): bump tomhjp/gh-action-jira-comment from 0.1.0 to 0.2.0 (#16684) Bumps [tomhjp/gh-action-jira-comment](https://github.com/tomhjp/gh-action-jira-comment) from 0.1.0 to 0.2.0. - [Release notes](https://github.com/tomhjp/gh-action-jira-comment/releases) - [Commits](https://github.com/tomhjp/gh-action-jira-comment/compare/v0.1.0...v0.2.0) --- updated-dependencies: - dependency-name: tomhjp/gh-action-jira-comment dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Yu * NET-2397: Add readme.md to upgrade test subdirectory (#16610) * NET-2397: Add readme.md to upgrade test subdirectory * remove test code * fix link and update steps of adding new test cases (#16654) * fix link and update steps of adding new test cases * Apply suggestions from code review Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> --------- Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> --------- Co-authored-by: cskh Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> * chore: replace hardcoded node name with a constant (#16692) * Fix broken links from api docs (#16695) * Update WAL Known issues (#16676) * UI: update Ember to 3.28.6 (#16616) --------- Co-authored-by: wenincode * Regen helm docs (#16701) * Remove unused are hosts set check (#16691) * Remove unused are hosts set check * Remove all traces of unused 'AreHostsSet' parameter * Remove unused Hosts attribute * Remove commented out use of snap.APIGateway.Hosts * [NET-3029] Migrate build-distros to GHA (#16669) * migrate build distros to GHA Signed-off-by: Dan Bond * build-arm Signed-off-by: Dan Bond * don't use matrix Signed-off-by: Dan Bond * check-go-mod Signed-off-by: Dan Bond * add notify slack script Signed-off-by: Dan Bond * notify slack if failure Signed-off-by: Dan Bond * rm notify slack script Signed-off-by: Dan Bond * fix check-go-mod job Signed-off-by: Dan Bond --------- Signed-off-by: Dan Bond * Update envoy extension docs, service-defaults, add multi-config example for lua (#16710) * fix build workflow (#16719) Signed-off-by: Dan Bond * Helm docs without developer.hashicorp.com prefix (#16711) This was causing linter errors * add extra resiliency to snapshot restore test (#16712) * fix: gracefully fail on invalid port number (#16721) * Copyright headers for config files git + circleci (#16703) * Copyright headers for config files git + circleci * Release folder copyright headers * fix bug where pqs that failover to a cluster peer dont un-fail over (#16729) * add enterprise xds tests (#16738) * delete config when nil (#16690) * delete config when nil * fix mock interface implementation * fix handler test to use the right assertion * extract DeleteConfig as a separate API. * fix mock limiter implementation to satisfy the new interface * fix failing tests * add test comments * Changelog for audit logging fix. (#16700) * Changelog for audit logging fix. * Use GH issues type for edu board (#16750) * fix: remove unused tenancy category from rate limit spec (#16740) * Remove version bump from CRT workflow (#16728) This bumps the version to reflect the next patch release; however, we use a specific branch for each patch release and so never wind up cutting a release directly from the `release/1.15.x` (for example) where this is intended to work. * tests instantiating clients w/o shutting down (#16755) noticed via their port still in use messages. * RELENG-471: Remove obsolete load-test workflow (#16737) * Remove obsolete load-test workflow * remove load-tests from circleci config. --------- Co-authored-by: John Murret * add failover policy to ProxyConfigEntry in api (#16759) * add failover policy to ProxyConfigEntry in api * update docs * Fix broken links in Consul docs (#16640) * Fix broken links in Consul docs * more broken link fixes * more 404 fixes * 404 fixes * broken link fix --------- Co-authored-by: Tu Nguyen * Change partition for peers in discovery chain targets (#16769) This commit swaps the partition field to the local partition for discovery chains targeting peers. Prior to this change, peer upstreams would always use a value of default regardless of which partition they exist in. This caused several issues in xds / proxycfg because of id mismatches. Some prior fixes were made to deal with one-off id mismatches that this PR also cleans up, since they are no longer needed. * Docs/intentions refactor docs day 2022 (#16758) * converted intentions conf entry to ref CT format * set up intentions nav * add page for intentions usage * final intentions usage page * final intentions overview page * fixed old relative links * updated diagram for overview * updated links to intentions content * fixed typo in updated links * rename intentions overview page file to index * rollback link updates to intentions overview * fixed nav * Updated custom HTML in API and CLI pages to MD * applied suggestions from review to index page * moved conf examples from usage to conf ref * missed custom HTML section * applied additional feedback * Apply suggestions from code review Co-authored-by: Tu Nguyen * updated headings in usage page * renamed files and udpated nav * updated links to new file names * added redirects and final tweaks * typo --------- Co-authored-by: Tu Nguyen * Add storage backend interface and in-memory implementation (#16538) Introduces `storage.Backend`, which will serve as the interface between the Resource Service and the underlying storage system (Raft today, but in the future, who knows!). The primary design goal of this interface is to keep its surface area small, and push as much functionality as possible into the layers above, so that new implementations can be added with little effort, and easily proven to be correct. To that end, we also provide a suite of "conformance" tests that can be run against a backend implementation to check it behaves correctly. In this commit, we introduce an initial in-memory storage backend, which is suitable for tests and when running Consul in development mode. This backend is a thin wrapper around the `Store` type, which implements a resource database using go-memdb and our internal pub/sub system. `Store` will also be used to handle reads in our Raft backend, and in the future, used as a local cache for external storage systems. * Fix bug in changelog checker where bash variable is not quoted (#16681) * Read(...) endpoint for the resource service (#16655) * Fix Edu Jira automation (#16778) * Fix struct tags for TCPService enterprise meta (#16781) * Fix struct tags for TCPService enterprise meta * Add changelog * Expand route flattening test for multiple namespaces (#16745) * Exand route flattening test for multiple namespaces * Add helper for checking http route config entry exists without checking for bound status * Fix port and hostname check for http route flattening test * WatchList(..) endpoint for the resource service (#16726) * Allocate virtual ip for resolver/router/splitter config entries (#16760) * add ip rate limiter controller OSS parts (#16790) * Resource service List(..) endpoint (#16753) * changes to support new PQ enterprise fields (#16793) * add scripts for testing locally consul-ui-toolkit (#16794) * Update normalization of route refs (#16789) * Use merge of enterprise meta's rather than new custom method * Add merge logic for tcp routes * Add changelog * Normalize certificate refs on gateways * Fix infinite call loop * Explicitly call enterprise meta * copyright headers for agent folder (#16704) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * Copyright headers for command folder (#16705) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * copyright headers for agent folder * Copyright headers for command folder * fix merge conflicts * Add copyright headers for acl, api and bench folders (#16706) * copyright headers for agent folder * Ignore test data files * fix proto files and remove headers in agent/uiserver folder * ignore deep-copy files * copyright headers for agent folder * fix merge conflicts * copyright headers for agent folder * Ignore test data files * fix proto files * ignore agent/uiserver folder for now * copyright headers for agent folder * Add copyright headers for acl, api and bench folders * Github Actions Migration - move go-tests workflows to GHA (#16761) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * making on pull_request * Update .github/scripts/rerun_fails_report.sh Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * make runs-on required * removing go-version param that is not used. * removing go-version param that is not used. * Modify build-distros to use medium runners (#16773) * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * trying mediums * adding in script * fixing runs-on to be parameter * fixing merge conflict * changing to on push * removing whitespace * go-tests workflow * add test splitting to go-tests * fix re-reun fails report path * fix re-reun fails report path another place * fixing tests for32bit and race * use script file to generate runners * fixing run path * add checkout * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * Apply suggestions from code review Co-authored-by: Dan Bond * passing runs-on * setting up runs-on as a parameter to check-go-mod * changing back to on pull_request --------- Co-authored-by: Dan Bond * Github Actions Migration - move verify-ci workflows to GHA (#16777) * add verify-ci workflow * adding comment and changing to on pull request. * changing to pull_requests * changing to pull_request * Apply suggestions from code review Co-authored-by: Dan Bond * [NET-3029] Migrate frontend to GHA (#16731) * changing set up to a small * using consuls own custom runner pool. --------- Co-authored-by: Dan Bond * Copyright headers for missing files/folders (#16708) * copyright headers for agent folder * fix: export ReadWriteRatesConfig struct as it needs to referenced from consul-k8s (#16766) * docs: Updates to support HCP Consul cluster peering release (#16774) * New HCP Consul documentation section + links * Establish cluster peering usage cross-link * unrelated fix to backport to v1.15 * nav correction + fixes * Tech specs fixes * specifications for headers * Tech specs fixes + alignments * sprawl edits * Tip -> note * port ENT ingress gateway upgrade tests [NET-2294] [NET-2296] (#16804) * [COMPLIANCE] Add Copyright and License Headers (#16807) * [COMPLIANCE] Add Copyright and License Headers * fix headers for generated files * ignore dist folder --------- Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> Co-authored-by: Ronald Ekambi Co-authored-by: Ronald * add order by locality failover to Consul enterprise (#16791) * ci: changes resulting from running on consul-enterprise (#16816) * changes resulting from running on consul-enterprise * removing comment line * port ENT upgrade tests flattening (#16824) * docs: raise awareness of GH-16779 (#16823) * updating command to reflect the additional package exclusions in CircleCI (#16829) * storage: fix resource leak in Watch (#16817) * Remove UI brand-loader copyright headers as they do not render appropriately (#16835) * Add sameness-group to exported-services config entries (#16836) This PR adds the sameness-group field to exported-service config entries, which allows for services to be exported to multiple destination partitions / peers easily. * Add default resolvers to disco chains based on the default sameness group (#16837) * [NET-3029] Migrate dev-* jobs to GHA (#16792) * ci: add build-artifacts workflow Signed-off-by: Dan Bond * makefile for gha dev-docker Signed-off-by: Dan Bond * use docker actions instead of make Signed-off-by: Dan Bond * Add context Signed-off-by: Dan Bond * testing push Signed-off-by: Dan Bond * set short sha Signed-off-by: Dan Bond * upload to s3 Signed-off-by: Dan Bond * rm s3 upload Signed-off-by: Dan Bond * use runner setup job Signed-off-by: Dan Bond * on push Signed-off-by: Dan Bond * testing Signed-off-by: Dan Bond * on pr Signed-off-by: Dan Bond * revert testing Signed-off-by: Dan Bond * OSS/ENT logic Signed-off-by: Dan Bond * add comments Signed-off-by: Dan Bond * Update .github/workflows/build-artifacts.yml Co-authored-by: John Murret --------- Signed-off-by: Dan Bond Co-authored-by: John Murret * add region field (#16825) * add region field * fix syntax error in test file * go fmt * go fmt * remove test * Connect CA Primary Provider refactor (#16749) * Rename Intermediate cert references to LeafSigningCert Within the Consul CA subsystem, the term "Intermediate" is confusing because the meaning changes depending on provider and datacenter (primary vs secondary). For example, when using the Consul CA the "ActiveIntermediate" may return the root certificate in a primary datacenter. At a high level, we are interested in knowing which CA is responsible for signing leaf certs, regardless of its position in a certificate chain. This rename makes the intent clearer. * Move provider state check earlier * Remove calls to GenerateLeafSigningCert GenerateLeafSigningCert (formerly known as GenerateIntermediate) is vestigial in non-Vault providers, as it simply returns the root certificate in primary datacenters. By folding Vault's intermediate cert logic into `GenerateRoot` we can encapsulate the intermediate cert handling within `newCARoot`. * Move GenerateLeafSigningCert out of PrimaryProvidder Now that the Vault Provider calls GenerateLeafSigningCert within GenerateRoot, we can remove the method from all other providers that never used it in a meaningful way. * Add test for IntermediatePEM * Rename GenerateRoot to GenerateCAChain "Root" was being overloaded in the Consul CA context, as different providers and configs resulted in a single root certificate or a chain originating from an external trusted CA. Since the Vault provider also generates intermediates, it seems more accurate to call this a CAChain. * Update changelog with patch releases (#16856) * Update changelog with patch releases * Backport missed 1.0.4 patch release to changelog * Fix typo on cli-flags.mdx (#16843) Change "segements" to segments * Allow dialer to re-establish terminated peering (#16776) Currently, if an acceptor peer deletes a peering the dialer's peering will eventually get to a "terminated" state. If the two clusters need to be re-peered the acceptor will re-generate the token but the dialer will encounter this error on the call to establish: "failed to get addresses to dial peer: failed to refresh peer server addresses, will continue to use initial addresses: there is no active peering for "<<>>"" This is because in `exchangeSecret().GetDialAddresses()` we will get an error if fetching addresses for an inactive peering. The peering shows up as inactive at this point because of the existing terminated state. Rather than checking whether a peering is active we can instead check whether it was deleted. This way users do not need to delete terminated peerings in the dialing cluster before re-establishing them. * CA mesh CA expiration to it's own section This is part of an effort to raise awareness that you need to monitor your mesh CA if coming from an external source as you'll need to manage the rotation. * Fix broken doc in consul-k8s upgrade (#16852) Signed-off-by: dttung2905 Co-authored-by: David Yu * docs: add envoy to the proxycfg diagram (#16834) * docs: add envoy to the proxycfg diagram * ci: increase deep-copy and lint-enum jobs to use large runner as they hang in ENT (#16866) * docs: add envoy to the proxycfg diagram (#16834) * docs: add envoy to the proxycfg diagram * increase dee-copy job to use large runner. disable lint-enums on ENT * set lint-enums to a large * remove redunant installation of deep-copy --------- Co-authored-by: cskh * Raft storage backend (#16619) * ad arm64 testing (#16876) * Omit false positives from 404 checker (#16881) * Remove false positives from 404 checker * fix remaining 404s * ci: fixes missing deps in frontend gha workflows (#16872) Signed-off-by: Dan Bond * always test oss and conditionally test enterprise (#16827) * temporarily disable macos-arm64 tests job in go-tests (#16898) * Resource `Write` endpoint (#16786) * Resource `Delete` endpoint (#16756) * Wasm Envoy HTTP extension (#16877) * Fix API GW broken link (#16885) * Fix API GW broken link * Update website/content/docs/api-gateway/upgrades.mdx Co-authored-by: Tu Nguyen --------- Co-authored-by: Tu Nguyen * ci: Add success jobs. make go-test-enterprise conditional. build-distros and go-tests trigger on push to main and release branches (#16905) * Add go-tests-success job and make go-test-enterprise conditional * fixing lint-32bit reference * fixing reference to -go-test-troubleshoot * add all jobs that fan out. * fixing success job to need set up * add echo to success job * adding success jobs to build-artifacts, build-distros, and frontend. * changing the name of the job in verify ci to be consistent with other workflows * enable go-tests, build-distros, and verify-ci to run on merge to main and release branches because they currently do not with just the pull_request trigger * increase ENT runner size for xl to match OSS. have guild-distros use xl to match CircleCI (#16920) * log warning about certificate expiring sooner and with more details The old setting of 24 hours was not enough time to deal with an expiring certificates. This change ups it to 28 days OR 40% of the full cert duration, whichever is shorter. It also adds details to the log message to indicate which certificate it is logging about and a suggested action. * highlight the agent.tls cert metric with CA ones Include server agent certificate with list of cert metrics that need monitoring. * docs: improve upgrade path guidance (#16925) * Test: add noCleanup to TestServer stop (#16919) * docs: fix typo in LocalRequestTimeoutMs (#16917) * ci: add GOTAGS to build-distros (#16934) * APIGW: Routes with duplicate parents should be invalid (#16926) * ensure route parents are unique when creating an http route * Ensure tcp route parents are unique * Added unit tests * ci: remove verify-ci from circleci (#16860) * ci: remove go-tests workflow from CircleCI (#16855) * remove go-tests workflow from CircleCI * add yaml anchor back * ci: build-artifacts - fix platform missing in manifest error (#16940) * ci: build-artifacts - fix platform missing in manifest error * remove platform key * Check acls on resource `Read`, `List`, and `WatchList` (#16842) * Resource validation hook for `Write` endpoint (#16950) * Remove deprecated service-defaults upstream behavior. (#16957) Prior to this change, peer services would be targeted by service-default overrides as long as the new `peer` field was not found in the config entry. This commit removes that deprecated backwards-compatibility behavior. Now it is necessary to specify the `peer` field in order for upstream overrides to apply to a peer upstream. * Fix the indentation of the copyAnnotations example (#16873) * Update docs for service-defaults overrides. (#16960) Update docs for service-defaults overrides. Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> * resource: `WriteStatus` endpoint (#16886) * Remove global.name requirement for APs (#16964) This is not a requirement when using APs because each AP has its own auth method so it's okay if the names overlap. * ci: remove build-distros from CircleCI (#16941) * feat: add reporting config with reload (#16890) * Added backport labels to PR template checklist (#16966) * ci: split frontend ember jobs (#16973) Signed-off-by: Dan Bond * Memdb Txn Commit race condition fix (#16871) * Add a test to reproduce the race condition * Fix race condition by publishing the event after the commit and adding a lock to prevent out of order events. * split publish to generate the list of events before committing the transaction. * add changelog * remove extra func * Apply suggestions from code review Co-authored-by: Dan Upton * add comment to explain test --------- Co-authored-by: Dan Upton * add sameness to exported services structs in the api package (#16984) * circleci: remove frontend jobs (#16906) * circleci: remove fronted jobs Signed-off-by: Dan Bond * remove frontend-cache Signed-off-by: Dan Bond --------- Signed-off-by: Dan Bond * Enforce ACLs on resource `Write` and `Delete` endpoints (#16956) * Update list of Envoy versions (#16889) * Update list of Envoy versions * Update docs + CI + tests * Add changelog entry * Add newly-released Envoy versions 1.23.8 and 1.24.6 * Add newly-released Envoy version 1.22.11 * Add mutate hook to `Write` endpoint (#16958) * upgrade test: config nodeName, nodeid, and inherited persistent data for consul container (#16931) * move enterprise test cases out of open source (#16985) * Fix delete when uid not provided (#16996) * Enforce Owner rules in `Write` endpoint (#16983) * add IP rate limiting config update (#16997) * add IP rate limiting config update * fix review comments * * added Sameness Group to proto files (#16998) - added Sameness Group to config entries - added Sameness Group to subscriptions * generated proto files * added Sameness Group events to the state store - added test cases * Refactored health RPC Client - moved code that is common to rpcclient under rpcclient common.go. This will help set us up to support future RPC clients * Refactored proxycfg glue views - Moved views to rpcclient config entry. This will allow us to reuse this code for a config entry client * added config entry RPC Client - Copied most of the testing code from rpcclient/health * hooked up new rpcclient in agent * fixed documentation and comments for clarity * added missing error message content to troubleshooting (#17005) * Add PrioritizeByLocality to config entries. (#17007) This commit adds the PrioritizeByLocality field to both proxy-config and service-resolver config entries for locality-aware routing. The field is currently intended for enterprise only, and will be used to enable prioritization of service-mesh connections to services based on geographical region / zone. * fixed bad link (#17009) * added an intro statement for the SI conf entry confiration model (#17017) * added an intro statement for the SI conf entry confiration model * caught a few more typos * Tenancy wildcard validaton for `Write`, `Read`, and `Delete` endpoints (#17004) * docs: update docs related to GH-16779 (#17020) * server: wire up in-process Resource Service (#16978) * add ability to start container tests in debug mode and attach a debugger (#16887) * add ability to start container tests in debug mode and attach a debugger to consul while running it. * add a debug message with the debug port * use pod to get the right port * fix image used in basic test * add more data to identify which container to debug. * fix comment Co-authored-by: Evan Culver * rename debugUri to debugURI --------- Co-authored-by: Evan Culver * feat: set up reporting agent (#16991) * api: enable query options on agent force-leave endpoint (#15987) * Bump the golang.org/x/net to 0.7.0 to address CVE-2022-41723 (#16754) * Bump the golang.org/x/net to 0.7.0 to address CVE-2022-41723 https://nvd.nist.gov/vuln/detail/CVE-2022-41723 * Add changelog entry --------- Co-authored-by: Nathan Coleman * Don't send updates twice (#16999) * ci: add test-integrations (#16915) * add test-integrations workflow * add test-integrations success job * update vault integration testing versions (#16949) * change parallelism to 4 forgotestsum. use env.CONSUL_VERSION so we can see the version. * use env for repeated values * match test to circleci * fix envvar * fix envvar 2 * fix envvar 3 * fix envvar 4 * fix envvar 5 * make upgrade and compatibility tests match circleci * run go env to check environment * debug docker Signed-off-by: Dan Bond * debug docker Signed-off-by: Dan Bond * revert debug docker Signed-off-by: Dan Bond * going back to command that worked 5 days ago for compatibility tests * Update Envoy versions to reflect changes in #16889 * cd to test dir * try running ubuntu latest * update PR with latest changes that work in enterprise * yaml still sucks * test GH fix (localhost resolution) * change for testing * test splitting and ipv6 lookup for compatibility and upgrade tests * fix indention * consul as image name * remove the on push * add gotestsum back in * removing the use of the gotestsum download action * yaml sucks today just like yesterday * fixing nomad tests * worked out the kinks on enterprise --------- Signed-off-by: Dan Bond Co-authored-by: John Eikenberry Co-authored-by: Dan Bond Co-authored-by: Nathan Coleman Co-authored-by: Sarah * ci: remove test-integrations CircleCI workflow (#16928) * remove all CircleCI files * remove references to CircleCI * remove more references to CircleCI * pin golangci-lint to v1.51.1 instead of v1.51 * Avoid decoding nil pointer in map walker (#17048) * Revert "cache: refactor agent cache fetching to prevent unnecessary f… (#16818) (#17046) Revert "cache: refactor agent cache fetching to prevent unnecessary fetches on error (#14956)" Co-authored-by: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> * Permissive mTLS (#17035) This implements permissive mTLS , which allows toggling services into "permissive" mTLS mode. Permissive mTLS mode allows incoming "non Consul-mTLS" traffic to be forward unmodified to the application. * Update service-defaults and proxy-defaults config entries with a MutualTLSMode field * Update the mesh config entry with an AllowEnablingPermissiveMutualTLS field and implement the necessary validation. AllowEnablingPermissiveMutualTLS must be true to allow changing to MutualTLSMode=permissive, but this does not require that all proxy-defaults and service-defaults are currently in strict mode. * Update xDS listener config to add a "permissive filter chain" when MutualTLSMode=permissive for a particular service. The permissive filter chain matches incoming traffic by the destination port. If the destination port matches the service port from the catalog, then no mTLS is required and the traffic sent is forwarded unmodified to the application. * [NET-3090] Add new JWT provider config entry (#17036) * [NET-3090] Add new JWT provider config entry * Add initial test cases * update validations for jwt-provider config entry fields * more validation * start improving tests * more tests * Normalize * Improve tests and move validate fns * usage test update * Add split between ent and oss for partitions * fix lint issues * Added retry backoff, fixed tests, removed unused defaults * take into account default partitions * use countTrue and add aliases * omit audiences if empty * fix failing tests * add omit-entry * update copyright headers ids --------- Co-authored-by: Ronald Ekambi Co-authored-by: Ronald * [NET-3091] Update service intentions to support jwt provider references (#17037) * [NET-3090] Add new JWT provider config entry * Add initial test cases * update validations for jwt-provider config entry fields * more validation * start improving tests * more tests * Normalize * Improve tests and move validate fns * usage test update * Add split between ent and oss for partitions * fix lint issues * Added retry backoff, fixed tests, removed unused defaults * take into account default partitions * use countTrue and add aliases * omit audiences if empty * fix failing tests * add omit-entry * Add JWT intentions * generate proto * fix deep copy issues * remove extra field * added some tests * more tests * add validation for creating existing jwt * fix nil issue * More tests, fix conflicts and improve memdb call * fix namespace * add aliases * consolidate errors, skip duplicate memdb calls * reworked iteration over config entries * logic improvements from review --------- Co-authored-by: Ronald Ekambi * remove worklogs upload (#17056) * [COMPLIANCE] Add Copyright and License Headers (#16854) Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> Co-authored-by: Ronald * Fix generated proto files (#17063) * [COMPLIANCE] Add Copyright and License Headers * generate proto --------- Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> * fix broken links (#17032) * fix broken links * Apply suggestions from code review Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --------- Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> * Add sameness groups to service intentions. (#17064) * Enforce operator:write acl on `WriteStatus` endpoint (#17019) * NET-3648: Add script to get consul and envoy version (#17060) * use proper TOTAL_RUNNER setting when generating runner matrix. if matrix size is smaller than total_runners, use the smaller number * try again * try again 2 * try again 3 * try again 4 * try again 5 * try scenario where number is less * backport of commit 4ca8f8c65c4fb1262ef70786549a8f9617d31816 * backport of commit 5185c5ada3ab41f9eca76c25acfdbcc764bceeef * backport of commit 171df26f9cb29ebfb3c30db8298a3666c12a41d6 * backport of commit a786025ed1bdbbf74e4e0138f4a750be79d4c2ea * backport of commit f36c71ca7633cbc42a9b82bad2c277378ae4a0f6 --------- Signed-off-by: dependabot[bot] Signed-off-by: Dan Bond Signed-off-by: dttung2905 Co-authored-by: R.B. Boyer <4903+rboyer@users.noreply.github.com> Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> Co-authored-by: Eddie Rowe <74205376+eddie-rowe@users.noreply.github.com> Co-authored-by: skpratt Co-authored-by: John Eikenberry Co-authored-by: David Yu Co-authored-by: Tu Nguyen Co-authored-by: cskh Co-authored-by: Ronald Co-authored-by: Nick Irvine <115657443+nfi-hashicorp@users.noreply.github.com> Co-authored-by: Chris S. Kim Co-authored-by: Michael Hofer Co-authored-by: Anita Akaeze Co-authored-by: Andrew Stucki Co-authored-by: Eric Haberkorn Co-authored-by: Michael Wilkerson <62034708+wilkermichael@users.noreply.github.com> Co-authored-by: Matt Keeler Co-authored-by: Melisa Griffin Co-authored-by: John Maguire Co-authored-by: Ashlee M Boyer <43934258+ashleemboyer@users.noreply.github.com> Co-authored-by: Valeriia Ruban Co-authored-by: Paul Glass Co-authored-by: Semir Patel Co-authored-by: Bryce Kalow Co-authored-by: Tyler Wendlandt Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> Co-authored-by: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Co-authored-by: Ashvitha Co-authored-by: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Co-authored-by: Bastien Dronneau Co-authored-by: Freddy Co-authored-by: Paul Banks Co-authored-by: wangxinyi7 <121973291+wangxinyi7@users.noreply.github.com> Co-authored-by: Vipin John Wilson <37441623+vjwilson1987@users.noreply.github.com> Co-authored-by: Rosemary Wang <915624+joatmon08@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Dhia Ayachi Co-authored-by: John Murret Co-authored-by: Poonam Jadhav Co-authored-by: Nitya Dhanushkodi Co-authored-by: Dan Bond Co-authored-by: Nathan Coleman Co-authored-by: brian shore Co-authored-by: malizz Co-authored-by: Dan Upton Co-authored-by: Kyle Havlovitz Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com> Co-authored-by: Ronald Ekambi Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> Co-authored-by: Michael Zalimeni Co-authored-by: Hariram Sankaran <56744845+ramramhariram@users.noreply.github.com> Co-authored-by: Dao Thanh Tung Co-authored-by: Chris Thain <32781396+cthain@users.noreply.github.com> Co-authored-by: Andrea Scarpino Co-authored-by: Thomas Eckert Co-authored-by: Evan Culver Co-authored-by: Andrei Komarov <15227837+andreikom@users.noreply.github.com> Co-authored-by: Kevin Wang <61252360+kevinwangcn@users.noreply.github.com> Co-authored-by: Sarah --- .github/workflows/test-integrations.yml | 42 +++++++++++++++++++------ 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 36f33c015bef..62a7813a9f4c 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -155,17 +155,25 @@ jobs: # multiplied by 8 based on these values: # envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"] # xds-target: ["server", "client"] - TOTAL_RUNNERS: 3 + TOTAL_RUNNERS: 4 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test/integration/connect/envoy -maxdepth 1 -type d | wc -l) + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$NUM_DIRS + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "envoy-matrix=" find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ | xargs -0 -n 1 basename \ - | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" - cat "$GITHUB_OUTPUT" envoy-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} @@ -261,18 +269,27 @@ jobs: - name: Generate Compatibility Job Matrix id: set-matrix env: - TOTAL_RUNNERS: 5 + + TOTAL_RUNNERS: 6 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | cd ./test/integration/consul-container + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test -maxdepth 2 -type d | wc -l) + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$NUM_DIRS + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "compatibility-matrix=" find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ | grep -v util | grep -v upgrade \ - | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join(" "))' } >> "$GITHUB_OUTPUT" - cat "$GITHUB_OUTPUT" + compatibility-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: @@ -359,17 +376,24 @@ jobs: - name: Generate Updgrade Job Matrix id: set-matrix env: - TOTAL_RUNNERS: 4 + TOTAL_RUNNERS: 5 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | cd ./test/integration/consul-container/test/upgrade + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' | wc -l) + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$NUM_DIRS + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "upgrade-matrix=" go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \ - | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" - cat "$GITHUB_OUTPUT" upgrade-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} From 7fbc68a4c46234abf9144eb1a4bc2ce166922752 Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Mon, 24 Apr 2023 17:04:16 +0100 Subject: [PATCH 181/421] Backport 1.15: Bump raft to 1.5.0 (#17081) (#17088) * Bump raft to 1.5.0 (#17081) * Bump raft to 1.5.0 * Add CHANGELOG entry * Go mod tidy * Fix release 1.15 GHA integrations yaml * Go mod tidy container tests --- .changelog/17081.txt | 3 +++ .github/workflows/test-integrations.yml | 1 - go.mod | 4 ++-- go.sum | 8 ++++---- test/integration/consul-container/go.mod | 9 ++------- test/integration/consul-container/go.sum | 14 ++++++-------- 6 files changed, 17 insertions(+), 22 deletions(-) create mode 100644 .changelog/17081.txt diff --git a/.changelog/17081.txt b/.changelog/17081.txt new file mode 100644 index 000000000000..5d17a3048473 --- /dev/null +++ b/.changelog/17081.txt @@ -0,0 +1,3 @@ +```release-note:improvement +Fixes a performance issue in Raft where commit latency can increase by 100x or more when under heavy load. For more details see https://github.com/hashicorp/raft/pull/541. +``` diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 62a7813a9f4c..2b1c26da14c9 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -170,7 +170,6 @@ jobs: echo -n "envoy-matrix=" find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ | xargs -0 -n 1 basename \ - | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" diff --git a/go.mod b/go.mod index 8e1a59773c05..1c3479fa1215 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-connlimit v0.3.0 github.com/hashicorp/go-discover v0.0.0-20220714221025-1c234a67149a - github.com/hashicorp/go-hclog v1.2.1 + github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-immutable-radix v1.3.1 github.com/hashicorp/go-memdb v1.3.4 github.com/hashicorp/go-multierror v1.1.1 @@ -62,7 +62,7 @@ require ( github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038 github.com/hashicorp/memberlist v0.5.0 - github.com/hashicorp/raft v1.4.0 + github.com/hashicorp/raft v1.5.0 github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/hashicorp/raft-wal v0.3.0 diff --git a/go.sum b/go.sum index 15c7db3b7949..49b0f9f4d38e 100644 --- a/go.sum +++ b/go.sum @@ -530,8 +530,8 @@ github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.2.1 h1:YQsLlGDJgwhXFpucSPyVbCBviQtjlHv3jLTlp8YmtEw= -github.com/hashicorp/go-hclog v1.2.1/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= @@ -608,8 +608,8 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56 github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= -github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= -github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= +github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8= +github.com/hashicorp/raft v1.5.0/go.mod h1:pKHB2mf/Y25u3AHNSXVRv+yT+WAnmeTX0BwVppVQV+M= github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 3eb170ccd5a7..aa5d7a7e02a8 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -21,14 +21,10 @@ require ( github.com/stretchr/testify v1.8.1 github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 github.com/testcontainers/testcontainers-go v0.15.0 - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 + golang.org/x/mod v0.8.0 ) require ( - fortio.org/dflag v1.5.2 // indirect - fortio.org/log v1.3.0 // indirect - fortio.org/sets v1.0.2 // indirect - fortio.org/version v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Microsoft/hcsshim v0.9.7 // indirect @@ -75,12 +71,11 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sirupsen/logrus v1.8.1 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/exp v0.0.0-20230303215020-44a13b063f3e // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect + google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum index 60a8a25fec66..65c18b26261f 100644 --- a/test/integration/consul-container/go.sum +++ b/test/integration/consul-container/go.sum @@ -119,7 +119,7 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -397,9 +397,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -934,8 +933,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1043,8 +1042,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1056,7 +1055,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From a10f09ccd13a1e1f33d561e51c7ae87714616cd1 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Mon, 24 Apr 2023 15:54:55 -0700 Subject: [PATCH 182/421] backport of commit 717d7fe7154971e4cdd246e7537ed6e48b034a2a (#17116) Co-authored-by: Maliz --- website/content/docs/k8s/k8s-cli.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/content/docs/k8s/k8s-cli.mdx b/website/content/docs/k8s/k8s-cli.mdx index bb45986e11e1..d0ad98c49a86 100644 --- a/website/content/docs/k8s/k8s-cli.mdx +++ b/website/content/docs/k8s/k8s-cli.mdx @@ -794,7 +794,6 @@ $ consul-k8s troubleshoot upstreams -pod | Flag | Description | Default | | ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | | `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | -| `-envoy-admin-endpoint` | `String` Envoy sidecar address and port | `127.0.0.1:19000` | #### Example Commands @@ -839,7 +838,6 @@ $ consul-k8s troubleshoot proxy -pod -upstream-envoy-id

    - 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -