Skip to content

Commit

Permalink
Migrate finalizer upgrade test to use the upgrade framework
Browse files Browse the repository at this point in the history
  • Loading branch information
skmatti committed Nov 20, 2019
1 parent 26ad076 commit a956f29
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 123 deletions.
91 changes: 0 additions & 91 deletions cmd/e2e-test/finalizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ package main

import (
"context"
"k8s.io/api/networking/v1beta1"
"k8s.io/ingress-gce/pkg/fuzz/features"
"testing"

"k8s.io/apimachinery/pkg/util/intstr"
Expand Down Expand Up @@ -195,92 +193,3 @@ func TestFinalizerIngressesWithSharedResources(t *testing.T) {
}
})
}

// TestUpdateTo1dot7 asserts that finalizer is added to an ingress when upgraded from a version
// without finalizer to version 1.7. Also, verifies that ingress is deleted with finalizer enabled.
// Note: The test is named in such a way that it does run as a normal test or an upgrade test for
// other versions.
func TestUpdateTo1dot7(t *testing.T) {
port80 := intstr.FromInt(80)
svcName := "service-1"
ing := fuzz.NewIngressBuilder("", "ingress-1", "").
AddPath("foo.com", "/", svcName, port80).
SetIngressClass("gce").
Build()
Framework.RunWithSandbox("finalizer-master-upgrade", t, func(t *testing.T, s *e2e.Sandbox) {
t.Parallel()

_, err := e2e.CreateEchoService(s, svcName, nil)
if err != nil {
t.Fatalf("CreateEchoService(_, %q, nil): %v, want nil", svcName, err)
}
t.Logf("Echo service created (%s/%s)", s.Namespace, svcName)

crud := e2e.IngressCRUD{C: Framework.Clientset}
ing.Namespace = s.Namespace
if _, err := crud.Create(ing); err != nil {
t.Fatalf("create(%s/%s) = %v, want nil; Ingress: %v", ing.Namespace, ing.Name, err, ing)
}
t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)
ing = common.WaitForStableIngress(true, ing, s, t)

// Check that finalizer is not added in old version in which finalizer add is not enabled.
ingFinalizers := ing.GetFinalizers()
if l := len(ingFinalizers); l != 0 {
t.Fatalf("GetFinalizers() = %d, want 0", l)
}
// Perform whitebox testing.
common.WhiteboxTest(ing, s, t, Framework.Cloud, "")

for {
// While k8s master is upgrading, it will return a connection refused
// error for any k8s resource we try to hit. We loop until the
// master upgrade has finished.
if s.MasterUpgrading() {
continue
}

if s.MasterUpgraded() {
t.Logf("Detected master upgrade")
break
}
}

// Wait for finalizer to be added and verify that correct finalizer is added to the ingress after the upgrade.
if err := e2e.WaitForFinalizer(s, ing.Name); err != nil {
t.Errorf("e2e.WaitForFinalizer(_, %q) = %v, want nil", ing.Name, err)
}

// Perform whitebox testing.
gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")

// If the Master has upgraded and the Ingress is stable,
// we delete the Ingress and exit out of the loop to indicate that
// the test is done.
deleteOptions := &fuzz.GCLBDeleteOptions{
SkipDefaultBackend: true,
}
if err := e2e.WaitForIngressDeletion(context.Background(), gclb, s, ing, deleteOptions); err != nil {
t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
}
})
}

func checkGCLB(t *testing.T, s *e2e.Sandbox, ing *v1beta1.Ingress, numForwardingRules, numBackendServices int) *fuzz.GCLB {
// Perform whitebox testing.
if len(ing.Status.LoadBalancer.Ingress) < 1 {
t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
}
vip := ing.Status.LoadBalancer.Ingress[0].IP
t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All)}
gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
if err != nil {
t.Fatalf("GCLBForVIP(..., %q, _) = %v, want nil; error getting GCP resources for LB with IP", vip, err)
}

if err = e2e.CheckGCLB(gclb, numForwardingRules, numBackendServices); err != nil {
t.Error(err)
}
return gclb
}
112 changes: 112 additions & 0 deletions cmd/e2e-test/upgrade/finalizer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrade

import (
"context"
"testing"

"k8s.io/api/networking/v1beta1"
"k8s.io/ingress-gce/cmd/e2e-test/common"
"k8s.io/ingress-gce/pkg/e2e"
"k8s.io/ingress-gce/pkg/fuzz"
)

// Finalizer implements e2e.UpgradeTest interface.
type Finalizer struct {
t *testing.T
s *e2e.Sandbox
framework *e2e.Framework
crud e2e.IngressCRUD
ing *v1beta1.Ingress
}

// NewFinalizerUpgradeTest returns an upgrade test that asserts that finalizer
// is added to an ingress when upgraded from a version without finalizer to v1.7.0.
// Also, verifies that ingress is deleted with finalizer enabled.
func NewFinalizerUpgradeTest() e2e.UpgradeTest {
return &Finalizer{}
}

// Name implements e2e.UpgradeTest.Init.
func (fr *Finalizer) Name() string {
return "FinalizerUpgrade"
}

// Init implements e2e.UpgradeTest.Init.
func (fr *Finalizer) Init(t *testing.T, s *e2e.Sandbox, framework *e2e.Framework) error {
fr.t = t
fr.s = s
fr.framework = framework
return nil
}

// PreUpgrade implements e2e.UpgradeTest.PreUpgrade.
func (fr *Finalizer) PreUpgrade() error {
_, err := e2e.CreateEchoService(fr.s, svcName, nil)
if err != nil {
fr.t.Fatalf("error creating echo service: %v", err)
}
fr.t.Logf("Echo service created (%s/%s)", fr.s.Namespace, svcName)

ing := fuzz.NewIngressBuilder(fr.s.Namespace, ingName, "").
AddPath("foo.com", "/", svcName, port80).
Build()
fr.crud = e2e.IngressCRUD{C: fr.framework.Clientset}
if _, err := fr.crud.Create(ing); err != nil {
fr.t.Fatalf("error creating Ingress %s/%s: %v", ing.Namespace, ing.Name, err)
}
fr.t.Logf("Ingress created (%s/%s)", fr.s.Namespace, ing.Name)

fr.ing = common.WaitForStableIngress(true, ing, fr.s, fr.t)
fr.t.Logf("GCLB resources created (%s/%s)", fr.s.Namespace, ing.Name)

// Check that finalizer is not added in old version in which finalizer add is not enabled.
ingFinalizers := ing.GetFinalizers()
if l := len(ingFinalizers); l != 0 {
fr.t.Fatalf("len(GetFinalizers()) = %d, want 0", l)
}

common.WhiteboxTest(ing, fr.s, fr.t, fr.framework.Cloud, "")
return nil
}

// DuringUpgrade implements e2e.UpgradeTest.DuringUpgrade.
func (fr *Finalizer) DuringUpgrade() error {
return nil
}

// PostUpgrade implements e2e.UpgradeTest.PostUpgrade
func (fr *Finalizer) PostUpgrade() error {
// Wait for finalizer to be added and verify that correct finalizer is added to the ingress after the upgrade.
if err := e2e.WaitForFinalizer(fr.s, fr.ing.Name); err != nil {
fr.t.Fatalf("e2e.WaitForFinalizer(_, %q) = %v, want nil", fr.ing.Name, err)
}
gclb := common.WhiteboxTest(fr.ing, fr.s, fr.t, fr.framework.Cloud, "")

// If the Master has upgraded and the Ingress is stable,
// we delete the Ingress and exit out of the loop to indicate that
// the test is done.
deleteOptions := &fuzz.GCLBDeleteOptions{
SkipDefaultBackend: true,
}

if err := e2e.WaitForIngressDeletion(context.Background(), gclb, fr.s, fr.ing, deleteOptions); err != nil {
fr.t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", fr.ing.Name, err)
}
return nil
}
81 changes: 49 additions & 32 deletions cmd/e2e-test/upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,43 +31,60 @@ func TestGenericUpgrade(t *testing.T) {
upgrade.NewStandaloneNegUpgradeTest(),
} {
test := test // Capture test as we are running this in parallel.
desc := test.Name()
Framework.RunWithSandbox(desc, t, func(t *testing.T, s *e2e.Sandbox) {
t.Parallel()
genericUpgrade(t, test)
}
}

t.Logf("Running upgrade test %v", desc)
if err := test.Init(t, s, Framework); err != nil {
t.Fatalf("For upgrade test %v, step Init failed due to %v", desc, err)
}
// TestUpgradeToV1dot7 runs upgrade tests for features that are introduced in v1.7.0.
// Note that this test runs only when an upgrade results in enabling these features.
func TestUpgradeToV1dot7(t *testing.T) {
t.Parallel()

s.PutStatus(e2e.Unstable)
func() {
// always mark the test as stable in order to unblock other upgrade tests.
defer s.PutStatus(e2e.Stable)
if err := test.PreUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step PreUpgrade failed due to %v", desc, err)
}
}()
for _, test := range []e2e.UpgradeTest{
upgrade.NewFinalizerUpgradeTest(),
} {
test := test // Capture test as we are running this in parallel.
genericUpgrade(t, test)
}
}

for {
// While k8s master is upgrading, it will return a connection refused
// error for any k8s resource we try to hit. We loop until the
// master upgrade has finished.
if s.MasterUpgrading() {
if err := test.DuringUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step DuringUpgrade failed due to %v", desc, err)
}
continue
}
func genericUpgrade(t *testing.T, test e2e.UpgradeTest) {
desc := test.Name()
Framework.RunWithSandbox(desc, t, func(t *testing.T, s *e2e.Sandbox) {
t.Parallel()

t.Logf("Running upgrade test %v", desc)
if err := test.Init(t, s, Framework); err != nil {
t.Fatalf("For upgrade test %v, step Init failed due to %v", desc, err)
}

if s.MasterUpgraded() {
t.Logf("Detected master upgrade, continuing upgrade test %v", desc)
break
s.PutStatus(e2e.Unstable)
func() {
// always mark the test as stable in order to unblock other upgrade tests.
defer s.PutStatus(e2e.Stable)
if err := test.PreUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step PreUpgrade failed due to %v", desc, err)
}
}()

for {
// While k8s master is upgrading, it will return a connection refused
// error for any k8s resource we try to hit. We loop until the
// master upgrade has finished.
if s.MasterUpgrading() {
if err := test.DuringUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step DuringUpgrade failed due to %v", desc, err)
}
continue
}
if err := test.PostUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step PostUpgrade failed due to %v", desc, err)

if s.MasterUpgraded() {
t.Logf("Detected master upgrade, continuing upgrade test %v", desc)
break
}
})
}
}
if err := test.PostUpgrade(); err != nil {
t.Fatalf("For upgrade test %v, step PostUpgrade failed due to %v", desc, err)
}
})
}

0 comments on commit a956f29

Please sign in to comment.