Onboard ACStor targets

.pipelines/azure-pipeline-build.yml

@@ -2,7 +2,7 @@ trigger:
   branches:
     include:
     - main
-
+    - shlok/acstor-onboarding
 pr:
   autoCancel: true
   branches:

otelcollector/configmapparser/default-prom-configs/acstorCapacityProvisionerDefaultFile.yml

@@ -0,0 +1,21 @@
+ scrape_configs:
+ - job_name: acstor-capacity-provisioner
+   honor_labels: true
+   scrape_interval: $$SCRAPE_INTERVAL$$
+   scheme: http
+   kubernetes_sd_configs:
+     - role: pod
+   relabel_configs:
+     # Include only specified namespace
+   - source_labels: [__meta_kubernetes_namespace]
+     action: keep
+     regex: acstor
+     # Include only specified pods
+   - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app_kubernetes_io_component]
+     action: keep
+     regex: capacity-provisioner;capacity-provisoner
+     # Include only specified ports
+   - source_labels: [__meta_kubernetes_pod_container_port_name]
+     action: keep
+     regex: metrics
+

otelcollector/configmapparser/default-prom-configs/acstorMetricsExporterDefaultFile.yml

@@ -0,0 +1,20 @@
+ scrape_configs:
+ - job_name: acstor-metrics-exporter
+   honor_labels: true
+   scrape_interval: $$SCRAPE_INTERVAL$$
+   scheme: http
+   kubernetes_sd_configs:
+     - role: pod
+   relabel_configs:
+     # Include only specified namespace
+   - source_labels: [__meta_kubernetes_namespace]
+     action: keep
+     regex: acstor
+     # Include only specified pods
+   - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app_kubernetes_io_component]
+     action: keep
+     regex: metrics-exporter;monitor
+     # Include only specified ports
+   - source_labels: [__meta_kubernetes_pod_container_port_name]
+     action: keep
+     regex: metrics

otelcollector/configmaps/ama-metrics-settings-configmap.yaml

@@ -29,6 +29,8 @@ data:
     controlplane-kube-scheduler = false
     controlplane-kube-controller-manager = false
     controlplane-etcd = true
+    acstor-capacity-provisioner = true
+    acstor-metrics-exporter = true
   # Regex for which namespaces to scrape through pod annotation based scraping.
   # This is none by default. Use '.*' to scrape all namespaces of annotated pods.
   pod-annotation-based-scraping: |-
@@ -53,6 +55,8 @@ data:
     controlplane-kube-scheduler = ""
     controlplane-kube-controller-manager = ""
     controlplane-etcd = ""
+    acstor-capacity-provisioner = ""
+    acstor-metrics-exporter = ""
     minimalingestionprofile = true
   default-targets-scrape-interval-settings: |-
     kubelet = "30s"
@@ -69,6 +73,8 @@ data:
     networkobservabilityHubble = "30s"
     networkobservabilityCilium = "30s"
     prometheuscollectorhealth = "30s"
+    acstor-capacity-provisioner = "30s"
+    acstor-metrics-exporter = "30s"
     podannotations = "30s"
   debug-mode: |-
     enabled = false

otelcollector/fluent-bit/src/telemetry.go

@@ -68,6 +68,11 @@ var (
 	PodannotationKeepListRegex string
 	// Kappie Basic metrics keep list regex
 	KappieBasicKeepListRegex string
+	// ACStor Capacity Provisioner keep list regex
+	AcstorCapacityProvisionerKeepListRegex string
+	// ACStor Metrics Exporter keep list regex
+	AcstorMetricsExporterKeepListRegex string
+
 	// Kubelet scrape interval
 	KubeletScrapeInterval string
 	// CoreDNS scrape interval
@@ -92,6 +97,11 @@ var (
 	PodAnnotationScrapeInterval string
 	// Kappie Basic scrape interval
 	KappieBasicScrapeInterval string
+	// ACStor Capacity Provisioner keep list regex
+	AcstorCapacityProvisionerScrapeInterval string
+	// ACStor Metrics Exporter keep list regex
+	AcstorMetricsExporterScrapeInterval string
+
 	// meMetricsProcessedCount map, which holds references to metrics per metric account
 	meMetricsProcessedCountMap = make(map[string]*meMetricsProcessedCount)
 	// meMetricsProcessedCountMapMutex -- used for reading & writing locks on meMetricsProcessedCountMap
@@ -290,6 +300,8 @@ func InitializeTelemetryClient(agentVersion string) (int, error) {
 			WinKubeProxyKeepListRegex = regexHash["WINDOWSKUBEPROXY_METRICS_KEEP_LIST_REGEX"]
 			PodannotationKeepListRegex = regexHash["POD_ANNOTATION_METRICS_KEEP_LIST_REGEX"]
 			KappieBasicKeepListRegex = regexHash["KAPPIEBASIC_METRICS_KEEP_LIST_REGEX"]
+			AcstorCapacityProvisionerKeepListRegex = regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"]
+			AcstorMetricsExporterKeepListRegex = regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"]
 		}
 	}
 
@@ -316,6 +328,8 @@ func InitializeTelemetryClient(agentVersion string) (int, error) {
 			PromHealthScrapeInterval = intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"]
 			PodAnnotationScrapeInterval = intervalHash["POD_ANNOTATION_SCRAPE_INTERVAL"]
 			KappieBasicScrapeInterval = intervalHash["KAPPIEBASIC_SCRAPE_INTERVAL"]
+			AcstorCapacityProvisionerScrapeInterval = intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"]
+			AcstorMetricsExporterScrapeInterval = intervalHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"]
 		}
 	}
 
@@ -689,6 +703,13 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() {
 				if KappieBasicKeepListRegex != "" {
 					metric.Properties["KappieBasicKeepListRegex"] = KappieBasicKeepListRegex
 				}
+				if AcstorCapacityProvisionerKeepListRegex != "" {
+					metric.Properties["AcstorCapacityProvisionerRegex"] = AcstorCapacityProvisionerKeepListRegex
+				}
+				if AcstorMetricsExporterKeepListRegex != "" {
+					metric.Properties["AcstorMetricsExporterRegex"] = AcstorMetricsExporterKeepListRegex
+				}
+
 				if KubeletScrapeInterval != "" {
 					metric.Properties["KubeletScrapeInterval"] = KubeletScrapeInterval
 				}
@@ -725,6 +746,12 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() {
 				if KappieBasicScrapeInterval != "" {
 					metric.Properties["KappieBasicScrapeInterval"] = KappieBasicScrapeInterval
 				}
+				if AcstorCapacityProvisionerScrapeInterval != "" {
+					metric.Properties["AcstorCapacityProvisionerScrapeInterval"] = AcstorCapacityProvisionerScrapeInterval
+				}
+				if AcstorMetricsExporterScrapeInterval != "" {
+					metric.Properties["AcstorMetricsExporterScrapeInterval"] = AcstorMetricsExporterScrapeInterval
+				}
 			}
 
 			TelemetryClient.Track(metric)

otelcollector/shared/configmap/mp/definitions.go

@@ -45,6 +45,8 @@ var (
 	networkObservabilityRetinaDefaultFileDs      = "networkobservabilityRetinaDefaultDs.yml"
 	networkObservabilityHubbleDefaultFileDs      = "networkobservabilityHubbleDefaultDs.yml"
 	networkObservabilityCiliumDefaultFileDs      = "networkobservabilityCiliumDefaultDs.yml"
+	acstorCapacityProvisionerDefaultFile         = "acstorCapacityProvisionerDefaultFile.yml"
+	acstorMetricsExporterDefaultFile             = "acstorMetricsExporterDefaultFile.yml"
 )
 
 type RegexValues struct {
@@ -63,6 +65,8 @@ type RegexValues struct {
 	networkobservabilitycilium string
 	podannotations             string
 	minimalingestionprofile    string
+	acstorcapacityprovisioner  string
+	acstormetricsexporter      string
 }
 
 // FilesystemConfigLoader implements ConfigLoader for file-based configuration loading.
@@ -99,6 +103,8 @@ type ConfigProcessor struct {
 	NetworkObservabilityRetina string
 	NetworkObservabilityHubble string
 	NetworkObservabilityCilium string
+	AcstorCapacityProvisioner  string
+	AcstorMetricsExporter      string
 }
 
 // ConfigParser is an interface for parsing configurations.

otelcollector/shared/configmap/mp/prometheus-config-merger.go

@@ -376,6 +376,7 @@ func populateDefaultPrometheusConfig() {
 	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_KUBESTATE_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType {
 		kubestateMetricsKeepListRegex, exists := regexHash["KUBESTATE_METRICS_KEEP_LIST_REGEX"]
 		kubestateScrapeInterval, intervalExists := intervalHash["KUBESTATE_SCRAPE_INTERVAL"]
+		log.Printf("path %s: %s\n", "kubeStateDefaultFile", kubeStateDefaultFile)
 
 		if intervalExists {
 			UpdateScrapeIntervalConfig(kubeStateDefaultFile, kubestateScrapeInterval)
@@ -662,6 +663,30 @@ func populateDefaultPrometheusConfig() {
 		}
 	}
 
+	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType {
+		acstorCapacityProvisionerKeepListRegex, exists := regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"]
+		acstorCapacityProvisionerScrapeInterval, intervalExists := intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"]
+		if intervalExists {
+			UpdateScrapeIntervalConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerScrapeInterval)
+		}
+		if exists && acstorCapacityProvisionerKeepListRegex != "" {
+			AppendMetricRelabelConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerKeepListRegex)
+		}
+		defaultConfigs = append(defaultConfigs, acstorCapacityProvisionerDefaultFile)
+	}
+
+	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType {
+		acstorMetricsExporterKeepListRegex, exists := regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"]
+		acstorMetricsExporterScrapeInterval, intervalExists := intervalHash["ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL"]
+		if intervalExists {
+			UpdateScrapeIntervalConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterScrapeInterval)
+		}
+		if exists && acstorMetricsExporterKeepListRegex != "" {
+			AppendMetricRelabelConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterKeepListRegex)
+		}
+		defaultConfigs = append(defaultConfigs, acstorMetricsExporterDefaultFile)
+	}
+
 	mergedDefaultConfigs = mergeDefaultScrapeConfigs(defaultConfigs)
 	// if mergedDefaultConfigs != nil {
 	// 	fmt.Printf("Merged default scrape targets: %v\n", mergedDefaultConfigs)
@@ -1099,6 +1124,32 @@ func populateDefaultPrometheusConfigWithOperator() {
 		}
 	}
 
+	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && (isConfigReaderSidecar() || currentControllerType == replicasetControllerType) {
+		acstorCapacityProvisionerKeepListRegex, exists := regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"]
+		acstorCapacityProvisionerScrapeInterval, intervalExists := intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"]
+		log.Printf("path %s: %s\n", "acstorCapacityProvisionerDefaultFile", acstorCapacityProvisionerDefaultFile)
+		if intervalExists {
+			UpdateScrapeIntervalConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerScrapeInterval)
+		}
+		if exists && acstorCapacityProvisionerKeepListRegex != "" {
+			AppendMetricRelabelConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerKeepListRegex)
+		}
+		defaultConfigs = append(defaultConfigs, acstorCapacityProvisionerDefaultFile)
+	}
+
+	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && (isConfigReaderSidecar() || currentControllerType == replicasetControllerType) {
+		acstorMetricsExporterKeepListRegex, exists := regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"]
+		acstorMetricsExporterScrapeInterval, intervalExists := intervalHash["ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL"]
+		log.Printf("path %s: %s\n", "acstorMetricsExporterDefaultFile", acstorMetricsExporterDefaultFile)
+		if intervalExists {
+			UpdateScrapeIntervalConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterScrapeInterval)
+		}
+		if exists && acstorMetricsExporterKeepListRegex != "" {
+			AppendMetricRelabelConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterKeepListRegex)
+		}
+		defaultConfigs = append(defaultConfigs, acstorMetricsExporterDefaultFile)
+	}
+
 	mergedDefaultConfigs = mergeDefaultScrapeConfigs(defaultConfigs)
 	// if mergedDefaultConfigs != nil {
 	// 	fmt.Printf("Merged default scrape targets: %v\n", mergedDefaultConfigs)
@@ -1217,7 +1268,7 @@ func setDefaultFileScrapeInterval(scrapeInterval string) {
 		prometheusCollectorHealthDefaultFile, windowsExporterDefaultRsSimpleFile, windowsExporterDefaultDsFile,
 		windowsKubeProxyDefaultFileRsSimpleFile, windowsKubeProxyDefaultDsFile, podAnnotationsDefaultFile,
 		kappieBasicDefaultFileDs, networkObservabilityRetinaDefaultFileDs, networkObservabilityHubbleDefaultFileDs,
-		networkObservabilityCiliumDefaultFileDs,
+		networkObservabilityCiliumDefaultFileDs, acstorMetricsExporterDefaultFile, acstorCapacityProvisionerDefaultFile,
 	}
 
 	for _, currentFile := range defaultFilesArray {

otelcollector/shared/configmap/mp/tomlparser-default-scrape-settings.go

@@ -25,6 +25,8 @@ func (fcl *FilesystemConfigLoader) SetDefaultScrapeSettings() (map[string]string
 	config["networkobservabilityHubble"] = "true"
 	config["networkobservabilityCilium"] = "true"
 	config["noDefaultsEnabled"] = "false"
+	config["acstor-capacity-provisioner"] = "true"
+	config["acstor-metrics-exporter"] = "true"
 	return config, nil
 }
 
@@ -46,6 +48,8 @@ func (fcl *FilesystemConfigLoader) ParseConfigMapForDefaultScrapeSettings() (map
 	config["networkobservabilityHubble"] = "true"
 	config["networkobservabilityCilium"] = "true"
 	config["noDefaultsEnabled"] = "false"
+	config["acstor-capacity-provisioner"] = "true"
+	config["acstor-metrics-exporter"] = "true"
 
 	if _, err := os.Stat(fcl.ConfigMapMountPath); os.IsNotExist(err) {
 		fmt.Println("configmap for default scrape settings not mounted, using defaults")
@@ -144,6 +148,16 @@ func (cp *ConfigProcessor) PopulateSettingValues(parsedConfig map[string]string)
 		fmt.Printf("config::Using scrape settings for networkobservabilityCilium: %v\n", cp.NetworkObservabilityCilium)
 	}
 
+	if val, ok := parsedConfig["acstor-capacity-provisioner"]; ok && val != "" {
+		cp.AcstorCapacityProvisioner = val
+		fmt.Printf("config:: Using scrape settings for acstor-capacity-provisioner: %v\n", cp.AcstorCapacityProvisioner)
+	}
+
+	if val, ok := parsedConfig["acstor-metrics-exporter"]; ok && val != "" {
+		cp.AcstorMetricsExporter = val
+		fmt.Printf("config:: Using scrape settings for acstor-metrics-exporter: %v\n", cp.AcstorMetricsExporter)
+	}
+
 	if os.Getenv("MODE") == "" && strings.ToLower(strings.TrimSpace(os.Getenv("MODE"))) == "advanced" {
 		controllerType := os.Getenv("CONTROLLER_TYPE")
 		if controllerType == "ReplicaSet" && strings.ToLower(os.Getenv("OS_TYPE")) == "linux" &&
@@ -183,6 +197,8 @@ func (fcw *FileConfigWriter) WriteDefaultScrapeSettingsToFile(filename string, c
 	file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED=%v\n", cp.NetworkObservabilityHubble))
 	file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED=%v\n", cp.NetworkObservabilityCilium))
 	file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NO_DEFAULT_SCRAPING_ENABLED=%v\n", cp.NoDefaultsEnabled))
+	file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED=%v\n", cp.AcstorCapacityProvisioner))
+	file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED=%v\n", cp.AcstorMetricsExporter))
 
 	return nil
 }