From 5488dd04f3dd9d801238da0071e8bfd19ac7541c Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Fri, 23 Jun 2023 09:02:51 -0700 Subject: [PATCH 1/3] clusterresolver: remove priority LB related tests --- .../balancer/clusterresolver/priority_test.go | 834 ------------------ 1 file changed, 834 deletions(-) delete mode 100644 xds/internal/balancer/clusterresolver/priority_test.go diff --git a/xds/internal/balancer/clusterresolver/priority_test.go b/xds/internal/balancer/clusterresolver/priority_test.go deleted file mode 100644 index 526204ca22c6..000000000000 --- a/xds/internal/balancer/clusterresolver/priority_test.go +++ /dev/null @@ -1,834 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package clusterresolver - -import ( - "context" - "errors" - "fmt" - "testing" - "time" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - "github.com/google/go-cmp/cmp" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/weightedtarget" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/balancergroup" - internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/xds/internal/balancer/clusterimpl" - "google.golang.org/grpc/xds/internal/balancer/priority" - "google.golang.org/grpc/xds/internal/balancer/wrrlocality" - xdstestutils "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeclient" - "google.golang.org/grpc/xds/internal/xdsclient" -) - -var ( - testClusterNames = []string{"test-cluster-1", "test-cluster-2"} - testSubZones = []string{"I", "II", "III", "IV"} - testEndpointAddrs []string - wrrLocalityLBConfig = &internalserviceconfig.BalancerConfig{ - Name: wrrlocality.Name, - Config: &wrrlocality.LBConfig{ - ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: "round_robin", - }, - }, - } -) - -const testBackendAddrsCount = 12 - -func init() { - for i := 0; i < testBackendAddrsCount; i++ { - testEndpointAddrs = append(testEndpointAddrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) - } - balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond - clusterimpl.NewRandomWRR = testutils.NewTestWRR - weightedtarget.NewRandomWRR = testutils.NewTestWRR - balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond * 100 -} - -func setupTestEDS(t *testing.T, initChild *internalserviceconfig.BalancerConfig) (balancer.Balancer, *testutils.TestClientConn, *fakeclient.Client, func()) { - xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar) - cc := testutils.NewTestClientConn(t) - builder := balancer.Get(Name) - edsb := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{URL: *testutils.MustParseURL("dns:///" + testEDSService)}}) - if edsb == nil { - t.Fatalf("builder.Build(%s) failed and returned nil", Name) - } - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := edsb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: xdsclient.SetClient(resolver.State{}, xdsC), - BalancerConfig: &LBConfig{ - DiscoveryMechanisms: []DiscoveryMechanism{{ - Cluster: testClusterName, - Type: DiscoveryMechanismTypeEDS, - }}, - xdsLBPolicy: *wrrLocalityLBConfig, - }, - }); err != nil { - edsb.Close() - t.Fatal(err) - } - if _, err := xdsC.WaitForWatchEDS(ctx); err != nil { - edsb.Close() - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - } - return edsb, cc, xdsC, func() { - edsb.Close() - } -} - -// When a high priority is ready, adding/removing lower locality doesn't cause -// changes. -// -// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0. -func (s) TestEDSPriority_HighPriorityReady(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - - // Two localities, with priorities [0, 1], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } - - // Add p2, it shouldn't cause any updates. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - - select { - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(defaultTestShortTimeout): - } - - select { - case p := <-cc.NewPickerCh: - // If we do get a new picker, ensure it is still a p1 picker. - if err := testutils.IsRoundRobin([]balancer.SubConn{sc1}, testutils.SubConnFromPicker(p)); err != nil { - t.Fatal(err) - } - default: - // No new picker; we were previously using p1 and should still be using - // p1, so this is okay. No need to wait for defaultTestShortTimeout - // since we just waited immediately above. - } - - // Remove p2, no updates. - clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil) - - select { - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(defaultTestShortTimeout): - } - - select { - case p := <-cc.NewPickerCh: - // If we do get a new picker, ensure it is still a p1 picker. - if err := testutils.IsRoundRobin([]balancer.SubConn{sc1}, testutils.SubConnFromPicker(p)); err != nil { - t.Fatal(err) - } - default: - // No new picker; we were previously using p1 and should still be using - // p1, so this is okay. No need to wait for defaultTestShortTimeout - // since we just waited immediately above. - } - -} - -// Lower priority is used when higher priority is not ready. -// -// Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is -// down, use 2; remove 2, use 1. -func (s) TestEDSPriority_SwitchPriority(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - - // Two localities, with priorities [0, 1], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { - t.Fatal(err) - } - - // Turn down 0, 1 is used. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with 1. - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } - - // Add p2, it shouldn't cause any updates. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - - select { - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(defaultTestShortTimeout): - } - - select { - case p := <-cc.NewPickerCh: - // If we do get a new picker, ensure it is still a p1 picker. - if err := testutils.IsRoundRobin([]balancer.SubConn{sc1}, testutils.SubConnFromPicker(p)); err != nil { - t.Fatal(err) - } - default: - // No new picker; we were previously using p1 and should still be using - // p1, so this is okay. No need to wait for defaultTestShortTimeout - // since we just waited immediately above. - } - - // Turn down 1, use 2 - scConnErr := errors.New("subConn connection error") - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ - ConnectivityState: connectivity.TransientFailure, - ConnectionError: scConnErr, - }) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with 2. - if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { - t.Fatal(err) - } - - // Remove 2, use 1. - clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil) - - // p2 SubConns are removed. - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove) - } - - // Should get an update with 1's old picker, to override 2's old picker. - want := errors.New("last connection error: subConn connection error") - if err := cc.WaitForPickerWithErr(ctx, want); err != nil { - t.Fatal(err) - } - -} - -// Add a lower priority while the higher priority is down. -// -// Init 0 and 1; 0 and 1 both down; add 2, use 2. -func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with different priorities, each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Turn down 0, 1 is used. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - // Turn down 1, pick should error. - scConnErr := errors.New("subConn connection error") - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ - ConnectivityState: connectivity.TransientFailure, - ConnectionError: scConnErr, - }) - - // Test pick failure. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - want := errors.New("last connection error: subConn connection error") - if err := cc.WaitForPickerWithErr(ctx, want); err != nil { - t.Fatal(err) - } - - // Add p2, it should create a new SubConn. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with 2. - if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { - t.Fatal(err) - } - -} - -// When a higher priority becomes available, all lower priorities are closed. -// -// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2. -func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with priorities [0,1,2], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab1.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Turn down 0, 1 is used. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - // Turn down 1, 2 is used. - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with 2. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { - t.Fatal(err) - } - - // When 0 becomes ready, 0 should be used, 1 and 2 should all be closed. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - var ( - scToRemove []balancer.SubConn - scToRemoveMap = make(map[balancer.SubConn]struct{}) - ) - // Each subconn is removed. - // The sub-balancer to be closed is priority's child, clusterimpl, who has - // weightedtarget as children. - // - When clusterimpl is closed, it closes weightedtarget, and this - // weightedtarget's balancer removes all the subconns. - for i := 0; i < 2; i++ { - // We expect 2 subconns, so we recv from channel 2 times. - scToRemoveMap[<-cc.RemoveSubConnCh] = struct{}{} - } - for sc := range scToRemoveMap { - scToRemove = append(scToRemove, sc) - } - - // sc1 and sc2 should be removed. - // - // With localities caching, the lower priorities are closed after a timeout, - // in goroutines. The order is no longer guaranteed. - if !(cmp.Equal(scToRemove[0], sc1, cmp.AllowUnexported(testutils.TestSubConn{})) && - cmp.Equal(scToRemove[1], sc2, cmp.AllowUnexported(testutils.TestSubConn{}))) && - !(cmp.Equal(scToRemove[0], sc2, cmp.AllowUnexported(testutils.TestSubConn{})) && - cmp.Equal(scToRemove[1], sc1, cmp.AllowUnexported(testutils.TestSubConn{}))) { - t.Errorf("RemoveSubConn, want [%v, %v], got %v", sc1, sc2, scToRemove) - } - - // Test pick with 0. - if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { - t.Fatal(err) - } -} - -// At init, start the next lower priority after timeout if the higher priority -// doesn't get ready. -// -// Init 0,1; 0 is not ready (in connecting), after timeout, use 1. -func (s) TestEDSPriority_InitTimeout(t *testing.T) { - const testPriorityInitTimeout = time.Second - defer func() func() { - old := priority.DefaultPriorityInitTimeout - priority.DefaultPriorityInitTimeout = testPriorityInitTimeout - return func() { - priority.DefaultPriorityInitTimeout = old - } - }()() - - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with different priorities, each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Keep 0 in connecting, 1 will be used after init timeout. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - - // Make sure new SubConn is created before timeout. - select { - case <-time.After(testPriorityInitTimeout * 3 / 4): - case <-cc.NewSubConnAddrsCh: - t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout") - } - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with 1. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } -} - -// Add localities to existing priorities. -// -// - start with 2 locality with p0 and p1 -// - add localities to existing p0 and p1 -func (s) TestEDSPriority_MultipleLocalities(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with different priorities, each with one backend. - clab0 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab0.Build()), nil) - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { - t.Fatal(err) - } - - // Turn down p0 subconns, p1 subconns will be created. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p1 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } - - // Reconnect p0 subconns, p1 subconn will be closed. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - - // Test roundrobin with only p0 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { - t.Fatal(err) - } - - // Add two localities, with two priorities, with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab1.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) - clab1.AddLocality(testSubZones[3], 1, 1, testEndpointAddrs[3:4], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only two p0 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc0, sc2); err != nil { - t.Fatal(err) - } - - // Turn down p0 subconns, p1 subconns will be created. - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - - sc3 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - sc4 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p1 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc3, sc4); err != nil { - t.Fatal(err) - } -} - -// EDS removes all localities, and re-adds them. -func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) { - const testPriorityInitTimeout = time.Second - defer func() func() { - old := priority.DefaultPriorityInitTimeout - priority.DefaultPriorityInitTimeout = testPriorityInitTimeout - return func() { - priority.DefaultPriorityInitTimeout = old - } - }()() - - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with different priorities, each with one backend. - clab0 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab0.Build()), nil) - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc0, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { - t.Fatal(err) - } - - // Remove all priorities. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - // p0 subconn should be removed. - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove) - } - - // time.Sleep(time.Second) - - // Test pick return TransientFailure. - if err := cc.WaitForPickerWithErr(ctx, priority.ErrAllPrioritiesRemoved); err != nil { - t.Fatal(err) - } - - // Re-add two localities, with previous priorities, but different backends. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[3:4], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - addrs01 := <-cc.NewSubConnAddrsCh - if got, want := addrs01[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc01 := <-cc.NewSubConnCh - - // Don't send any update to p0, so to not override the old state of p0. - // Later, connect to p1 and then remove p1. This will fallback to p0, and - // will send p0's old picker if they are not correctly removed. - - // p1 will be used after priority init timeout. - addrs11 := <-cc.NewSubConnAddrsCh - if got, want := addrs11[0].Addr, testEndpointAddrs[3]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc11 := <-cc.NewSubConnCh - edsb.UpdateSubConnState(sc11, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc11, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p1 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc11); err != nil { - t.Fatal(err) - } - - // Remove p1 from EDS, to fallback to p0. - clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil) - - // p1 subconn should be removed. - scToRemove1 := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove1, sc11, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc11, scToRemove1) - } - - // Test pick return TransientFailure. - if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { - t.Fatal(err) - } - - // Send an ready update for the p0 sc that was received when re-adding - // localities to EDS. - edsb.UpdateSubConnState(sc01, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc01, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc01); err != nil { - t.Fatal(err) - } - - select { - case <-cc.NewPickerCh: - t.Fatalf("got unexpected new picker") - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(defaultTestShortTimeout): - } -} - -// Test the case where the high priority contains no backends. The low priority -// will be used. -func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with priorities [0, 1], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } - - // Remove addresses from priority 0, should use p1. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, nil, nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - // p0 will remove the subconn, and ClientConn will send a sc update to - // shutdown. - scToRemove := <-cc.RemoveSubConnCh - edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - - // p1 is ready. - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p1 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { - t.Fatal(err) - } -} - -// Test the case where the high priority contains no healthy backends. The low -// priority will be used. -func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) { - edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // Two localities, with priorities [0, 1], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p0 subconns. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { - t.Fatal(err) - } - - // Set priority 0 endpoints to all unhealthy, should use p1. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], &xdstestutils.AddLocalityOptions{ - Health: []corepb.HealthStatus{corepb.HealthStatus_UNHEALTHY}, - }) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - // p0 will remove the subconn, and ClientConn will send a sc update to - // transient failure. - scToRemove := <-cc.RemoveSubConnCh - edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - - // p1 is ready. - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin with only p1 subconns. - if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { - t.Fatal(err) - } -} - -// Test the case where the first and only priority is removed. -func (s) TestEDSPriority_FirstPriorityRemoved(t *testing.T) { - const testPriorityInitTimeout = time.Second - defer func() func() { - old := priority.DefaultPriorityInitTimeout - priority.DefaultPriorityInitTimeout = testPriorityInitTimeout - return func() { - priority.DefaultPriorityInitTimeout = old - } - }()() - - _, cc, xdsC, cleanup := setupTestEDS(t, nil) - defer cleanup() - // One localities, with priorities [0], each with one backend. - clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) - // Remove the only localities. - clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := cc.WaitForErrPicker(ctx); err != nil { - t.Fatal(err) - } -} From 6b0ab5531e37b041df0dde3a6266c7d67d2fb1dd Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Fri, 23 Jun 2023 10:17:19 -0700 Subject: [PATCH 2/3] make vet happy by deleting more stuff --- .../balancer/clusterresolver/testutil_test.go | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 xds/internal/balancer/clusterresolver/testutil_test.go diff --git a/xds/internal/balancer/clusterresolver/testutil_test.go b/xds/internal/balancer/clusterresolver/testutil_test.go deleted file mode 100644 index 2792f802b258..000000000000 --- a/xds/internal/balancer/clusterresolver/testutil_test.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package clusterresolver - -import ( - "fmt" - "net" - "strconv" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" - typepb "github.com/envoyproxy/go-control-plane/envoy/type" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" -) - -// parseEDSRespProtoForTesting parses EDS response, and panic if parsing fails. -// -// TODO: delete this. The EDS balancer tests should build an EndpointsUpdate -// directly, instead of building and parsing a proto message. -func parseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) xdsresource.EndpointsUpdate { - u, err := parseEDSRespProto(m) - if err != nil { - panic(err.Error()) - } - return u -} - -// parseEDSRespProto turns EDS response proto message to EndpointsUpdate. -func parseEDSRespProto(m *xdspb.ClusterLoadAssignment) (xdsresource.EndpointsUpdate, error) { - ret := xdsresource.EndpointsUpdate{} - for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { - ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy)) - } - priorities := make(map[uint32]struct{}) - for _, locality := range m.Endpoints { - l := locality.GetLocality() - if l == nil { - return xdsresource.EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) - } - lid := internal.LocalityID{ - Region: l.Region, - Zone: l.Zone, - SubZone: l.SubZone, - } - priority := locality.GetPriority() - priorities[priority] = struct{}{} - ret.Localities = append(ret.Localities, xdsresource.Locality{ - ID: lid, - Endpoints: parseEndpoints(locality.GetLbEndpoints()), - Weight: locality.GetLoadBalancingWeight().GetValue(), - Priority: priority, - }) - } - for i := 0; i < len(priorities); i++ { - if _, ok := priorities[uint32(i)]; !ok { - return xdsresource.EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) - } - } - return ret, nil -} - -func parseAddress(socketAddress *corepb.SocketAddress) string { - return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) -} - -func parseDropPolicy(dropPolicy *xdspb.ClusterLoadAssignment_Policy_DropOverload) xdsresource.OverloadDropConfig { - percentage := dropPolicy.GetDropPercentage() - var ( - numerator = percentage.GetNumerator() - denominator uint32 - ) - switch percentage.GetDenominator() { - case typepb.FractionalPercent_HUNDRED: - denominator = 100 - case typepb.FractionalPercent_TEN_THOUSAND: - denominator = 10000 - case typepb.FractionalPercent_MILLION: - denominator = 1000000 - } - return xdsresource.OverloadDropConfig{ - Category: dropPolicy.GetCategory(), - Numerator: numerator, - Denominator: denominator, - } -} - -func parseEndpoints(lbEndpoints []*endpointpb.LbEndpoint) []xdsresource.Endpoint { - endpoints := make([]xdsresource.Endpoint, 0, len(lbEndpoints)) - for _, lbEndpoint := range lbEndpoints { - endpoints = append(endpoints, xdsresource.Endpoint{ - HealthStatus: xdsresource.EndpointHealthStatus(lbEndpoint.GetHealthStatus()), - Address: parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()), - Weight: lbEndpoint.GetLoadBalancingWeight().GetValue(), - }) - } - return endpoints -} From cde2a3e348db544c4e34ada8e4942cde198e3365 Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Fri, 23 Jun 2023 12:35:18 -0700 Subject: [PATCH 3/3] review comments --- xds/internal/balancer/priority/balancer_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go index 5e29edf6698a..22ecca84bf25 100644 --- a/xds/internal/balancer/priority/balancer_test.go +++ b/xds/internal/balancer/priority/balancer_test.go @@ -149,6 +149,11 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) { case <-time.After(time.Millisecond * 100): } + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + // Remove p2, no updates. if err := pb.UpdateClientConnState(balancer.ClientConnState{ ResolverState: resolver.State{ @@ -175,6 +180,11 @@ func (s) TestPriority_HighPriorityReady(t *testing.T) { t.Fatalf("got unexpected remove SubConn") case <-time.After(time.Millisecond * 100): } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } } // Lower priority is used when higher priority is not ready.