Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up stale resources when 'antctl check cluster' failed #6597

Merged
merged 1 commit into from
Aug 21, 2024
+18 −6
Merged

Clean up stale resources when 'antctl check cluster' failed #6597

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pkg/antctl/raw/check/cluster/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func Run(o *options) error {
}
ctx := context.Background()
testContext := NewTestContext(client, config, clusterName, o.testImage)
defer check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace)
if err := testContext.setup(ctx); err != nil {
return err
}
Expand All @@ -121,7 +122,6 @@ func Run(o *options) error {
}
}
testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were uncertain", numSuccess, numFailure, numUncertain)
check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace)
if numFailure > 0 {
return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry))
}
Expand Down Expand Up @@ -205,7 +205,7 @@ func (t *testContext) setup(ctx context.Context) error {
}
testPods, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "component=cluster-checker"})
if err != nil {
return fmt.Errorf("no pod found for test Deployment")
return fmt.Errorf("no Pod found for Deployment %s", deploymentName)
}
t.testPod = &testPods.Items[0]
return nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/antctl/raw/check/installation/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,13 +148,13 @@ func Run(o *options) error {
}
ctx := context.Background()
testContext := NewTestContext(client, config, clusterName, o.antreaNamespace, runFilterRegex, o.testImage)
defer check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace)
if err := testContext.setup(ctx); err != nil {
return err
}
stats := testContext.runTests(ctx)

testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were skipped", stats.numSuccess, stats.numFailure, stats.numSkipped)
check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace)
if stats.numFailure > 0 {
return fmt.Errorf("%v/%v tests failed", stats.numFailure, stats.numTotal())
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/antctl/raw/check/installation/test_denyall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package installation
import (
"context"
"fmt"
"time"

corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
Expand All @@ -27,6 +28,9 @@ type DenyAllConnectivityTest struct {
networkPolicy *networkingv1.NetworkPolicy
}

// Provide enough time for policies to be enforced by the CNI plugin.
const networkPolicyDelay = 2 * time.Second

func init() {
RegisterTest("egress-deny-all-connectivity", &DenyAllConnectivityTest{networkPolicy: &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -84,6 +88,7 @@ func (t *DenyAllConnectivityTest) Run(ctx context.Context, testContext *testCont
testContext.Log("NetworkPolicy deletion was successful")
return nil
}()
time.Sleep(networkPolicyDelay)
Copy link
Contributor Author

@luolanzone luolanzone Aug 9, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The *-deny-all e2e failures should have nothing to do with the changes for the stale resources cleanup, it's likely to be an issue with NP reconciliation which caused the service is connectable when deny is expected. So I added 2 seconds here to wait NP to be reconciled before the testing.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer something poll-based, but I guess this can be addressed in a later PR

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I can check later for poll-based with a new PR. Thanks.

testContext.Log("NetworkPolicy applied successfully")
for _, clientPod := range testContext.clientPods {
for _, service := range services {
Expand Down
13 changes: 10 additions & 3 deletions pkg/antctl/raw/check/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -216,12 +217,18 @@ func GenerateRandomNamespace(baseName string) string {

func Teardown(ctx context.Context, client kubernetes.Interface, clusterName string, namespace string) {
fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting installation tests setup...\n")
client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
if err != nil {
fmt.Fprintf(os.Stdout, "Namespace %s deletion failed: %v", namespace, err)
return
}
fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to be deleted\n", namespace)
err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
_, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
if err != nil {
return true, nil
if errors.IsNotFound(err) {
return true, nil
}
}
return false, nil
})
Expand Down