WIP: allow host collector progress in kots

Makefile

@@ -7,7 +7,7 @@ LVP_TAG ?= v0.6.7
 PACT_PUBLISH_CONTRACT ?= false
 
 OS ?= linux
-ARCH ?= $(shell go env GOARCH)
+ARCH ?= $(shell go env GOARCH) 
 
 .PHONY: test
 test:

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/replicatedhq/embedded-cluster/kinds v1.15.0
 	github.com/replicatedhq/kotskinds v0.0.0-20240718194123-1018dd404e95
 	github.com/replicatedhq/kurlkinds v1.5.0
-	github.com/replicatedhq/troubleshoot v0.107.1
+	github.com/replicatedhq/troubleshoot v0.107.2
 	github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq
 	github.com/robfig/cron v1.2.0
 	github.com/robfig/cron/v3 v3.0.1

go.sum

@@ -1352,6 +1352,8 @@ github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851 h1:eRlNDH
 github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851/go.mod h1:JDxG6+uubnk9/BZ2yUsyAJJwlptjrnmB2MPF5d2Xe/8=
 github.com/replicatedhq/troubleshoot v0.107.1 h1:Hx9VbVv1r3M5fiH2fPTeoZ8LNIxh5R/e6vpe2jBgPfc=
 github.com/replicatedhq/troubleshoot v0.107.1/go.mod h1:6mZzcO/EWVBNXVnFdSHfPaoTnjcQdV3sq61NkBF60YE=
+github.com/replicatedhq/troubleshoot v0.107.2 h1:KPMQR+inoNACvZE5AV/6teK4jcM+lFlH0vGZANmIU4g=
+github.com/replicatedhq/troubleshoot v0.107.2/go.mod h1:yzIVQsTu6bK+aw34SdWWUfh1UBhVwkDm6q1pVyoN6do=
 github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq h1:PwPggruelq2336c1Ayg5STFqgbn/QB1tWLQwrVlU7ZQ=
 github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq/go.mod h1:Txa7LopbYCU8aRgmNe0n+y/EPMz50NbCPcVVJBquwag=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

pkg/kotsutil/troubleshoot.go

@@ -2,6 +2,7 @@ package kotsutil
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 
@@ -50,3 +51,26 @@ func LoadTSKindsFromPath(dir string) (*troubleshootloader.TroubleshootKinds, err
 	}
 	return tsKinds, nil
 }
+
+func Dedup[T any](objs []T) []T {
+	seen := make(map[string]bool)
+	out := []T{}
+
+	if len(objs) == 0 {
+		return objs
+	}
+
+	for _, o := range objs {
+		data, err := json.Marshal(o)
+		if err != nil {
+			out = append(out, o)
+			continue
+		}
+		key := string(data)
+		if _, ok := seen[key]; !ok {
+			out = append(out, o)
+			seen[key] = true
+		}
+	}
+	return out
+}

pkg/supportbundle/execute.go

@@ -26,10 +26,11 @@ type supportBundleProgressUpdate struct {
 type supportBundleProgressUpdateType string
 
 const (
-	BUNDLE_PROGRESS_ERROR     supportBundleProgressUpdateType = "error"
-	BUNDLE_PROGRESS_COLLECTOR supportBundleProgressUpdateType = "collector"
-	BUNDLE_PROGRESS_FILETREE  supportBundleProgressUpdateType = "filetree"
-	BUNDLE_PROGRESS_UPLOADED  supportBundleProgressUpdateType = "uploaded"
+	BUNDLE_PROGRESS_ERROR          supportBundleProgressUpdateType = "error"
+	BUNDLE_PROGRESS_COLLECTOR      supportBundleProgressUpdateType = "collector"
+	BUNDLE_PROGRESS_HOST_COLLECTOR supportBundleProgressUpdateType = "collect.CollectProgress"
+	BUNDLE_PROGRESS_FILETREE       supportBundleProgressUpdateType = "filetree"
+	BUNDLE_PROGRESS_UPLOADED       supportBundleProgressUpdateType = "uploaded"
 )
 
 // supportBundleIsComplete checks the current progress against the declared totals in
@@ -78,6 +79,7 @@ func executeUpdateRoutine(bundle *types.SupportBundle) chan interface{} {
 
 				case supportBundleProgressUpdate:
 					// Collect events are saved separately since there are many
+
 					if val.Type == BUNDLE_PROGRESS_COLLECTOR {
 						logger.Debugf("Received collector progress update %d, %s, for support bundle ID: %s", collectorsComplete, val.Message, bundle.ID)
 
@@ -90,6 +92,18 @@ func executeUpdateRoutine(bundle *types.SupportBundle) chan interface{} {
 							return
 						}
 
+						// Host collectors are saved separately since there are many
+					} else if val.Type == BUNDLE_PROGRESS_HOST_COLLECTOR {
+						logger.Debugf("Received host collector progress update %d, %s, for support bundle ID: %s", collectorsComplete, val.Message, bundle.ID)
+
+						bundle.Progress.CollectorsCompleted = collectorsComplete
+						bundle.Progress.Message = val.Message
+
+						if err := store.GetStore().UpdateSupportBundle(bundle); err != nil {
+							logger.Error(errors.Wrap(err, "could not update progress for bundle"))
+							return
+						}
+
 						// Something went wrong and the loop finished
 					} else if val.Type == BUNDLE_PROGRESS_ERROR {
 						logger.Debugf("Received error in progress update, %s, for support bundle ID: %s", val.Message, bundle.ID)
@@ -144,6 +158,10 @@ func executeUpdateRoutine(bundle *types.SupportBundle) chan interface{} {
 	return progressChan
 }
 
+func typeof(msg interface{}) {
+	panic("unimplemented")
+}
+
 // executeSupportBundleCollectRoutine creates a goroutine to collect the support bundle, upload, analyze and
 // send redactors. The function takes a channel for progress updates and closes it when collectors are complete.
 func executeSupportBundleCollectRoutine(bundle *types.SupportBundle, progressChan chan interface{}) {
@@ -171,9 +189,11 @@ func executeSupportBundleCollectRoutine(bundle *types.SupportBundle, progressCha
 		Namespace:                 "",
 		ProgressChan:              progressChan,
 		Redact:                    true,
+		RunHostCollectorsInPod:    true, // always run host collectors in pod from KOTS regardless of the spec value
 	}
 
 	logger.Infof("Executing Collection go routine for support bundle ID: %s", bundle.ID)
+	logger.Infof("Always run host collectors in pod: %t", opts.RunHostCollectorsInPod)
 
 	go func() {
 		defer close(progressChan)

pkg/supportbundle/spec.go

@@ -163,11 +163,15 @@ func mergeSupportBundleSpecs(builtBundles map[string]*troubleshootv1beta2.Suppor
 		mergedBundle.Spec.Collectors = append(mergedBundle.Spec.Collectors, builtBundle.Spec.Collectors...)
 		mergedBundle.Spec.Analyzers = append(mergedBundle.Spec.Analyzers, builtBundle.Spec.Analyzers...)
 		mergedBundle.Spec.AfterCollection = append(mergedBundle.Spec.AfterCollection, builtBundle.Spec.AfterCollection...)
+		mergedBundle.Spec.HostCollectors = append(mergedBundle.Spec.HostCollectors, builtBundle.Spec.HostCollectors...)
+		mergedBundle.Spec.HostAnalyzers = append(mergedBundle.Spec.HostAnalyzers, builtBundle.Spec.HostAnalyzers...)
 	}
 
-	mergedBundle = deduplicatedCollectors(mergedBundle)
-	mergedBundle = deduplicatedAnalyzers(mergedBundle)
-	mergedBundle = deduplicatedAfterCollection(mergedBundle)
+	mergedBundle.Spec.Collectors = kotsutil.Dedup(mergedBundle.Spec.Collectors)
+	mergedBundle.Spec.Analyzers = kotsutil.Dedup(mergedBundle.Spec.Analyzers)
+	mergedBundle.Spec.AfterCollection = kotsutil.Dedup(mergedBundle.Spec.AfterCollection)
+	mergedBundle.Spec.HostCollectors = kotsutil.Dedup(mergedBundle.Spec.HostCollectors)
+	mergedBundle.Spec.HostAnalyzers = kotsutil.Dedup(mergedBundle.Spec.HostAnalyzers)
 
 	return mergedBundle
 }
@@ -465,11 +469,15 @@ func addDiscoveredSpecs(
 
 		supportBundle.Spec.Collectors = append(supportBundle.Spec.Collectors, sbObject.Spec.Collectors...)
 		supportBundle.Spec.Analyzers = append(supportBundle.Spec.Analyzers, sbObject.Spec.Analyzers...)
+		supportBundle.Spec.HostCollectors = append(supportBundle.Spec.HostCollectors, sbObject.Spec.HostCollectors...)
+		supportBundle.Spec.HostAnalyzers = append(supportBundle.Spec.HostAnalyzers, sbObject.Spec.HostAnalyzers...)
 	}
 
-	// remove duplicated collectors and analyzers if there are multiple support bundle upstream spec
-	supportBundle = deduplicatedCollectors(supportBundle)
-	supportBundle = deduplicatedAnalyzers(supportBundle)
+	// remove duplicated specs if there are multiple support bundle upstream spec
+	supportBundle.Spec.Collectors = kotsutil.Dedup(supportBundle.Spec.Collectors)
+	supportBundle.Spec.Analyzers = kotsutil.Dedup(supportBundle.Spec.Analyzers)
+	supportBundle.Spec.HostCollectors = kotsutil.Dedup(supportBundle.Spec.HostCollectors)
+	supportBundle.Spec.HostAnalyzers = kotsutil.Dedup(supportBundle.Spec.HostAnalyzers)
 
 	return supportBundle
 }

pkg/supportbundle/spec_test.go

@@ -769,3 +769,80 @@ func createNamespaces(t *testing.T, clientset kubernetes.Interface, namespaces .
 		require.NoError(t, err)
 	}
 }
+
+func Test_mergeSupportBundleSpecs(t *testing.T) {
+	testBundle := &troubleshootv1beta2.SupportBundle{
+		Spec: troubleshootv1beta2.SupportBundleSpec{
+			Collectors: []*troubleshootv1beta2.Collect{
+				{
+					ClusterResources: &troubleshootv1beta2.ClusterResources{
+						CollectorMeta: troubleshootv1beta2.CollectorMeta{CollectorName: "first"},
+					},
+				},
+				{
+					ClusterResources: &troubleshootv1beta2.ClusterResources{
+						CollectorMeta: troubleshootv1beta2.CollectorMeta{CollectorName: "first"},
+					},
+				},
+				{
+					ClusterResources: &troubleshootv1beta2.ClusterResources{},
+				},
+			},
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{CheckName: "first"},
+					},
+				},
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{CheckName: "first"},
+					},
+				},
+			},
+			AfterCollection: []*troubleshootv1beta2.AfterCollection{},
+			HostCollectors: []*troubleshootv1beta2.HostCollect{
+				{
+					CPU: &troubleshootv1beta2.CPU{},
+					Memory: &troubleshootv1beta2.Memory{
+						HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{CollectorName: "first"},
+					},
+				},
+				{
+					CPU: &troubleshootv1beta2.CPU{},
+					Memory: &troubleshootv1beta2.Memory{
+						HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{CollectorName: "first"},
+					},
+				},
+				{
+					CPU: &troubleshootv1beta2.CPU{},
+					Memory: &troubleshootv1beta2.Memory{
+						HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{CollectorName: "second"},
+					},
+				},
+			},
+			HostAnalyzers: []*troubleshootv1beta2.HostAnalyze{
+				{
+					CPU: &troubleshootv1beta2.CPUAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{CheckName: "first"},
+					},
+				},
+				{
+					CPU: &troubleshootv1beta2.CPUAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{CheckName: "first"},
+					},
+				},
+			},
+		},
+	}
+
+	builtBundles := map[string]*troubleshootv1beta2.SupportBundle{
+		"first": testBundle,
+	}
+	merged := mergeSupportBundleSpecs(builtBundles)
+
+	assert.Equal(t, 2, len(merged.Spec.Collectors))
+	assert.Equal(t, 1, len(merged.Spec.Analyzers))
+	assert.Equal(t, 2, len(merged.Spec.HostCollectors))
+	assert.Equal(t, 1, len(merged.Spec.HostAnalyzers))
+}

pkg/supportbundle/supportbundle.go

@@ -155,7 +155,7 @@ func CreateSupportBundleDependencies(app *apptypes.App, sequence int64, opts typ
 		URI:        GetSpecURI(app.GetSlug()),
 		RedactURIs: redactURIs,
 		Progress: types.SupportBundleProgress{
-			CollectorCount: len(supportBundle.Spec.Collectors),
+			CollectorCount: len(supportBundle.Spec.Collectors) + len(supportBundle.Spec.HostCollectors),
 		},
 	}