e2e: add hook to debug OIDC fail (envoyproxy#3914)

internal/cmd/egctl/collect.go

@@ -15,14 +15,12 @@ import (
 	"syscall"
 	"time"
 
-	troubleshootv1b2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
-	tbcollect "github.com/replicatedhq/troubleshoot/pkg/collect"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
 	"github.com/spf13/cobra"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 
 	"github.com/envoyproxy/gateway/internal/cmd/options"
-	"github.com/envoyproxy/gateway/internal/troubleshoot/collect"
+	tb "github.com/envoyproxy/gateway/internal/troubleshoot"
 )
 
 type collectOptions struct {
@@ -65,16 +63,6 @@ func runCollect(collectOpts collectOptions) error {
 	defer cancel()
 	go waitForSignal(ctx, cancel)
 
-	progressChan := make(chan interface{})
-	go func() {
-		select {
-		case <-ctx.Done():
-			close(progressChan)
-		case msg := <-progressChan:
-			fmt.Printf("Collecting support bundle: %v\n", msg)
-		}
-	}()
-
 	tmpDir, err := os.MkdirTemp("", "envoy-gateway-support-bundle")
 	if err != nil {
 		return fmt.Errorf("create temp dir: %w", err)
@@ -100,68 +88,8 @@ func runCollect(collectOpts collectOptions) error {
 		return fmt.Errorf("create bundle dir: %w", err)
 	}
 
-	var result tbcollect.CollectorResult
-	collectors := []tbcollect.Collector{
-		// Collect the custom resources from Gateway API and EG
-		collect.CustomResource{
-			ClientConfig: restConfig,
-			BundlePath:   bundlePath,
-			IncludeGroups: []string{
-				"gateway.envoyproxy.io",
-				"gateway.networking.k8s.io",
-			},
-		},
-		// Collect resources from EnvoyGateway system namespace
-		collect.EnvoyGatewayResource{
-			ClientConfig: restConfig,
-			BundlePath:   bundlePath,
-			Namespace:    collectOpts.envoyGatewayNamespace,
-		},
-		// Collect logs from EnvoyGateway system namespace
-		&tbcollect.CollectLogs{
-			Collector: &troubleshootv1b2.Logs{
-				Name:      "pod-logs",
-				Namespace: collectOpts.envoyGatewayNamespace,
-			},
-			ClientConfig: restConfig,
-			BundlePath:   bundlePath,
-			Context:      ctx,
-		},
-		// Collect prometheus metrics from EnvoyGateway system namespace
-		collect.PrometheusMetric{
-			BundlePath:   bundlePath,
-			ClientConfig: restConfig,
-			Namespace:    collectOpts.envoyGatewayNamespace,
-		},
-		// Collect config dump from EnvoyGateway system namespace
-		collect.ConfigDump{
-			BundlePath:   bundlePath,
-			ClientConfig: restConfig,
-			Namespace:    collectOpts.envoyGatewayNamespace,
-		},
-	}
-	total := len(collectors)
-	allCollectedData := make(map[string][]byte)
-	for i, collector := range collectors {
-		res, err := collector.Collect(progressChan)
-		if err != nil {
-			progressChan <- fmt.Errorf("failed to run collector: %s: %w", collector.Title(), err)
-			progressChan <- tbcollect.CollectProgress{
-				CurrentName:    collector.Title(),
-				CurrentStatus:  "failed",
-				CompletedCount: i + 1,
-				TotalCount:     total,
-			}
-			continue
-		}
-		for k, v := range res {
-			allCollectedData[k] = v
-		}
-	}
-	result = allCollectedData
-
-	filename := fmt.Sprintf("%s.tar.gz", basename)
-	return result.ArchiveSupportBundle(bundlePath, filename)
+	result := tb.CollectResult(ctx, restConfig, bundlePath, collectOpts.envoyGatewayNamespace)
+	return result.ArchiveSupportBundle(bundlePath, fmt.Sprintf("%s.tar.gz", basename))
 }
 
 func waitForSignal(c context.Context, cancel context.CancelFunc) {

internal/troubleshoot/collect.go

@@ -0,0 +1,92 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package troubleshoot
+
+import (
+	"context"
+	"fmt"
+
+	troubleshootv1b2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	tbcollect "github.com/replicatedhq/troubleshoot/pkg/collect"
+	"k8s.io/client-go/rest"
+
+	"github.com/envoyproxy/gateway/internal/troubleshoot/collect"
+)
+
+func CollectResult(ctx context.Context, restConfig *rest.Config, bundlePath string, egNamespace string) tbcollect.CollectorResult {
+	var result tbcollect.CollectorResult
+
+	progressChan := make(chan interface{})
+	go func() {
+		select {
+		case <-ctx.Done():
+			close(progressChan)
+		case msg := <-progressChan:
+			fmt.Printf("Collecting support bundle: %v\n", msg)
+		}
+	}()
+
+	collectors := []tbcollect.Collector{
+		// Collect the custom resources from Gateway API and EG
+		collect.CustomResource{
+			ClientConfig: restConfig,
+			BundlePath:   bundlePath,
+			IncludeGroups: []string{
+				"gateway.envoyproxy.io",
+				"gateway.networking.k8s.io",
+			},
+		},
+		// Collect resources from EnvoyGateway system namespace
+		collect.EnvoyGatewayResource{
+			ClientConfig: restConfig,
+			BundlePath:   bundlePath,
+			Namespace:    egNamespace,
+		},
+		// Collect logs from EnvoyGateway system namespace
+		&tbcollect.CollectLogs{
+			Collector: &troubleshootv1b2.Logs{
+				Name:      "pod-logs",
+				Namespace: egNamespace,
+			},
+			ClientConfig: restConfig,
+			BundlePath:   bundlePath,
+			Context:      ctx,
+		},
+		// Collect prometheus metrics from EnvoyGateway system namespace
+		collect.PrometheusMetric{
+			BundlePath:   bundlePath,
+			ClientConfig: restConfig,
+			Namespace:    egNamespace,
+		},
+		// Collect config dump from EnvoyGateway system namespace
+		collect.ConfigDump{
+			BundlePath:   bundlePath,
+			ClientConfig: restConfig,
+			Namespace:    egNamespace,
+		},
+	}
+	total := len(collectors)
+	allCollectedData := make(map[string][]byte)
+	for i, collector := range collectors {
+		res, err := collector.Collect(progressChan)
+		if err != nil {
+			progressChan <- fmt.Errorf("failed to run collector: %s: %w", collector.Title(), err)
+			progressChan <- tbcollect.CollectProgress{
+				CurrentName:    collector.Title(),
+				CurrentStatus:  "failed",
+				CompletedCount: i + 1,
+				TotalCount:     total,
+			}
+			continue
+		}
+		for k, v := range res {
+			allCollectedData[k] = v
+		}
+	}
+	result = allCollectedData
+
+	return result
+}

test/benchmark/benchmark_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestBenchmark(t *testing.T) {
-	cli := kubetest.NewClient(t)
+	cli, _ := kubetest.NewClient(t)
 
 	// Parse benchmark options.
 	flag.Parse()

test/e2e/e2e_test.go

@@ -26,7 +26,7 @@ import (
 func TestE2E(t *testing.T) {
 	flag.Parse()
 
-	c := kubetest.NewClient(t)
+	c, cfg := kubetest.NewClient(t)
 
 	if flags.RunTest != nil && *flags.RunTest != "" {
 		tlog.Logf(t, "Running E2E test %s with %s GatewayClass\n cleanup: %t\n debug: %t",
@@ -38,6 +38,7 @@ func TestE2E(t *testing.T) {
 
 	cSuite, err := suite.NewConformanceTestSuite(suite.ConformanceOptions{
 		Client:               c,
+		RestConfig:           cfg,
 		GatewayClassName:     *flags.GatewayClassName,
 		Debug:                *flags.ShowDebug,
 		CleanupBaseResources: *flags.CleanupBaseResources,

test/e2e/merge_gateways/merge_gateways_test.go

@@ -28,7 +28,7 @@ import (
 func TestMergeGateways(t *testing.T) {
 	flag.Parse()
 
-	c := kubetest.NewClient(t)
+	c, cfg := kubetest.NewClient(t)
 
 	if flags.RunTest != nil && *flags.RunTest != "" {
 		tlog.Logf(t, "Running E2E test %s with %s GatewayClass\n cleanup: %t\n debug: %t",
@@ -40,6 +40,7 @@ func TestMergeGateways(t *testing.T) {
 
 	cSuite, err := suite.NewConformanceTestSuite(suite.ConformanceOptions{
 		Client:               c,
+		RestConfig:           cfg,
 		GatewayClassName:     *flags.GatewayClassName,
 		Debug:                *flags.ShowDebug,
 		CleanupBaseResources: *flags.CleanupBaseResources,

test/e2e/multiple_gc/multiple_gc_test.go

@@ -28,7 +28,7 @@ import (
 func TestMultipleGC(t *testing.T) {
 	flag.Parse()
 
-	c := kubetest.NewClient(t)
+	c, cfg := kubetest.NewClient(t)
 
 	if flags.RunTest != nil && *flags.RunTest != "" {
 		tlog.Logf(t, "Running E2E test %s with %s GatewayClass\n cleanup: %t\n debug: %t",
@@ -43,6 +43,7 @@ func TestMultipleGC(t *testing.T) {
 		internetGatewaySuiteGatewayClassName := "internet"
 		internetGatewaySuite, err := suite.NewConformanceTestSuite(suite.ConformanceOptions{
 			Client:               c,
+			RestConfig:           cfg,
 			GatewayClassName:     internetGatewaySuiteGatewayClassName,
 			Debug:                *flags.ShowDebug,
 			CleanupBaseResources: *flags.CleanupBaseResources,

test/e2e/tests/oidc.go

@@ -50,6 +50,10 @@ var OIDCTest = suite.ConformanceTest{
 	Manifests:   []string{"testdata/oidc-keycloak.yaml", "testdata/oidc-securitypolicy.yaml"},
 	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
 		t.Run("http route with oidc authentication", func(t *testing.T) {
+			// Add a function to dump current cluster status
+			t.Cleanup(func() {
+				CollectAndDump(t, suite.RestConfig)
+			})
 			ns := "gateway-conformance-infra"
 			routeNN := types.NamespacedName{Name: "http-with-oidc", Namespace: ns}
 			gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}

test/e2e/tests/utils.go

@@ -33,13 +33,15 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	"sigs.k8s.io/gateway-api/conformance/utils/config"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 	"sigs.k8s.io/gateway-api/conformance/utils/tlog"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	tb "github.com/envoyproxy/gateway/internal/troubleshoot"
 )
 
 const defaultServiceStartupTimeout = 5 * time.Minute
@@ -569,3 +571,13 @@ func createTagsQueryParam(tags map[string]string) (string, error) {
 	}
 	return tagsBuilder.String(), nil
 }
+
+// CollectAndDump collects and dumps the cluster data for troubleshooting and log.
+// This function should be call within t.Cleanup.
+func CollectAndDump(t *testing.T, rest *rest.Config) {
+	result := tb.CollectResult(context.TODO(), rest, "", "envoy-gateway")
+	for r, data := range result {
+		tlog.Logf(t, "filename: %s", r)
+		tlog.Logf(t, "data: \n%s", data)
+	}
+}

test/e2e/upgrade/eg_upgrade_test.go

@@ -27,7 +27,7 @@ import (
 func TestEGUpgrade(t *testing.T) {
 	flag.Parse()
 
-	c := kubetest.NewClient(t)
+	c, cfg := kubetest.NewClient(t)
 
 	if flags.RunTest != nil && *flags.RunTest != "" {
 		tlog.Logf(t, "Running E2E test %s with %s GatewayClass\n cleanup: %t\n debug: %t",
@@ -39,6 +39,7 @@ func TestEGUpgrade(t *testing.T) {
 
 	cSuite, err := suite.NewConformanceTestSuite(suite.ConformanceOptions{
 		Client:               c,
+		RestConfig:           cfg,
 		GatewayClassName:     *flags.GatewayClassName,
 		Debug:                *flags.ShowDebug,
 		CleanupBaseResources: *flags.CleanupBaseResources,

test/utils/kubernetes/client.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
@@ -20,7 +21,7 @@ import (
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 )
 
-func NewClient(t *testing.T) client.Client {
+func NewClient(t *testing.T) (client.Client, *rest.Config) {
 	cfg, err := config.GetConfig()
 	require.NoError(t, err)
 
@@ -30,7 +31,7 @@ func NewClient(t *testing.T) client.Client {
 	// Install all the scheme to kubernetes client.
 	CheckInstallScheme(t, c)
 
-	return c
+	return c, cfg
 }
 
 func CheckInstallScheme(t *testing.T, c client.Client) {