diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index 364c43d2..69a753df 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -2,7 +2,7 @@ trigger: branches: include: - main - + - shlok/acstor-onboarding pr: autoCancel: true branches: diff --git a/otelcollector/configmapparser/default-prom-configs/acstorCapacityProvisionerDefaultFile.yml b/otelcollector/configmapparser/default-prom-configs/acstorCapacityProvisionerDefaultFile.yml new file mode 100644 index 00000000..ab38dc2f --- /dev/null +++ b/otelcollector/configmapparser/default-prom-configs/acstorCapacityProvisionerDefaultFile.yml @@ -0,0 +1,21 @@ + scrape_configs: + - job_name: acstor-capacity-provisioner + honor_labels: true + scrape_interval: $$SCRAPE_INTERVAL$$ + scheme: http + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Include only specified namespace + - source_labels: [__meta_kubernetes_namespace] + action: keep + regex: acstor + # Include only specified pods + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app_kubernetes_io_component] + action: keep + regex: capacity-provisioner;capacity-provisoner + # Include only specified ports + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: metrics + \ No newline at end of file diff --git a/otelcollector/configmapparser/default-prom-configs/acstorMetricsExporterDefaultFile.yml b/otelcollector/configmapparser/default-prom-configs/acstorMetricsExporterDefaultFile.yml new file mode 100644 index 00000000..b145857c --- /dev/null +++ b/otelcollector/configmapparser/default-prom-configs/acstorMetricsExporterDefaultFile.yml @@ -0,0 +1,20 @@ + scrape_configs: + - job_name: acstor-metrics-exporter + honor_labels: true + scrape_interval: $$SCRAPE_INTERVAL$$ + scheme: http + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Include only specified namespace + - source_labels: [__meta_kubernetes_namespace] + action: keep + regex: acstor + # Include only specified pods + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app_kubernetes_io_component] + action: keep + regex: metrics-exporter;monitor + # Include only specified ports + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: metrics diff --git a/otelcollector/configmaps/ama-metrics-settings-configmap.yaml b/otelcollector/configmaps/ama-metrics-settings-configmap.yaml index 2dfa6231..19736f35 100644 --- a/otelcollector/configmaps/ama-metrics-settings-configmap.yaml +++ b/otelcollector/configmaps/ama-metrics-settings-configmap.yaml @@ -29,6 +29,8 @@ data: controlplane-kube-scheduler = false controlplane-kube-controller-manager = false controlplane-etcd = true + acstor-capacity-provisioner = true + acstor-metrics-exporter = true # Regex for which namespaces to scrape through pod annotation based scraping. # This is none by default. Use '.*' to scrape all namespaces of annotated pods. pod-annotation-based-scraping: |- @@ -53,6 +55,8 @@ data: controlplane-kube-scheduler = "" controlplane-kube-controller-manager = "" controlplane-etcd = "" + acstor-capacity-provisioner = "" + acstor-metrics-exporter = "" minimalingestionprofile = true default-targets-scrape-interval-settings: |- kubelet = "30s" @@ -69,6 +73,8 @@ data: networkobservabilityHubble = "30s" networkobservabilityCilium = "30s" prometheuscollectorhealth = "30s" + acstor-capacity-provisioner = "30s" + acstor-metrics-exporter = "30s" podannotations = "30s" debug-mode: |- enabled = false diff --git a/otelcollector/fluent-bit/src/telemetry.go b/otelcollector/fluent-bit/src/telemetry.go index 329dba36..e3af87cf 100644 --- a/otelcollector/fluent-bit/src/telemetry.go +++ b/otelcollector/fluent-bit/src/telemetry.go @@ -68,6 +68,11 @@ var ( PodannotationKeepListRegex string // Kappie Basic metrics keep list regex KappieBasicKeepListRegex string + // ACStor Capacity Provisioner keep list regex + AcstorCapacityProvisionerKeepListRegex string + // ACStor Metrics Exporter keep list regex + AcstorMetricsExporterKeepListRegex string + // Kubelet scrape interval KubeletScrapeInterval string // CoreDNS scrape interval @@ -92,6 +97,11 @@ var ( PodAnnotationScrapeInterval string // Kappie Basic scrape interval KappieBasicScrapeInterval string + // ACStor Capacity Provisioner keep list regex + AcstorCapacityProvisionerScrapeInterval string + // ACStor Metrics Exporter keep list regex + AcstorMetricsExporterScrapeInterval string + // meMetricsProcessedCount map, which holds references to metrics per metric account meMetricsProcessedCountMap = make(map[string]*meMetricsProcessedCount) // meMetricsProcessedCountMapMutex -- used for reading & writing locks on meMetricsProcessedCountMap @@ -290,6 +300,8 @@ func InitializeTelemetryClient(agentVersion string) (int, error) { WinKubeProxyKeepListRegex = regexHash["WINDOWSKUBEPROXY_METRICS_KEEP_LIST_REGEX"] PodannotationKeepListRegex = regexHash["POD_ANNOTATION_METRICS_KEEP_LIST_REGEX"] KappieBasicKeepListRegex = regexHash["KAPPIEBASIC_METRICS_KEEP_LIST_REGEX"] + AcstorCapacityProvisionerKeepListRegex = regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"] + AcstorMetricsExporterKeepListRegex = regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"] } } @@ -316,6 +328,8 @@ func InitializeTelemetryClient(agentVersion string) (int, error) { PromHealthScrapeInterval = intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"] PodAnnotationScrapeInterval = intervalHash["POD_ANNOTATION_SCRAPE_INTERVAL"] KappieBasicScrapeInterval = intervalHash["KAPPIEBASIC_SCRAPE_INTERVAL"] + AcstorCapacityProvisionerScrapeInterval = intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"] + AcstorMetricsExporterScrapeInterval = intervalHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"] } } @@ -689,6 +703,13 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() { if KappieBasicKeepListRegex != "" { metric.Properties["KappieBasicKeepListRegex"] = KappieBasicKeepListRegex } + if AcstorCapacityProvisionerKeepListRegex != "" { + metric.Properties["AcstorCapacityProvisionerRegex"] = AcstorCapacityProvisionerKeepListRegex + } + if AcstorMetricsExporterKeepListRegex != "" { + metric.Properties["AcstorMetricsExporterRegex"] = AcstorMetricsExporterKeepListRegex + } + if KubeletScrapeInterval != "" { metric.Properties["KubeletScrapeInterval"] = KubeletScrapeInterval } @@ -725,6 +746,12 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() { if KappieBasicScrapeInterval != "" { metric.Properties["KappieBasicScrapeInterval"] = KappieBasicScrapeInterval } + if AcstorCapacityProvisionerScrapeInterval != "" { + metric.Properties["AcstorCapacityProvisionerScrapeInterval"] = AcstorCapacityProvisionerScrapeInterval + } + if AcstorMetricsExporterScrapeInterval != "" { + metric.Properties["AcstorMetricsExporterScrapeInterval"] = AcstorMetricsExporterScrapeInterval + } } TelemetryClient.Track(metric) diff --git a/otelcollector/shared/configmap/mp/definitions.go b/otelcollector/shared/configmap/mp/definitions.go index 8721a269..8f0503db 100644 --- a/otelcollector/shared/configmap/mp/definitions.go +++ b/otelcollector/shared/configmap/mp/definitions.go @@ -45,6 +45,8 @@ var ( networkObservabilityRetinaDefaultFileDs = "networkobservabilityRetinaDefaultDs.yml" networkObservabilityHubbleDefaultFileDs = "networkobservabilityHubbleDefaultDs.yml" networkObservabilityCiliumDefaultFileDs = "networkobservabilityCiliumDefaultDs.yml" + acstorCapacityProvisionerDefaultFile = "acstorCapacityProvisionerDefaultFile.yml" + acstorMetricsExporterDefaultFile = "acstorMetricsExporterDefaultFile.yml" ) type RegexValues struct { @@ -63,6 +65,8 @@ type RegexValues struct { networkobservabilitycilium string podannotations string minimalingestionprofile string + acstorcapacityprovisioner string + acstormetricsexporter string } // FilesystemConfigLoader implements ConfigLoader for file-based configuration loading. @@ -99,6 +103,8 @@ type ConfigProcessor struct { NetworkObservabilityRetina string NetworkObservabilityHubble string NetworkObservabilityCilium string + AcstorCapacityProvisioner string + AcstorMetricsExporter string } // ConfigParser is an interface for parsing configurations. diff --git a/otelcollector/shared/configmap/mp/prometheus-config-merger.go b/otelcollector/shared/configmap/mp/prometheus-config-merger.go index 42431438..a7fee07f 100644 --- a/otelcollector/shared/configmap/mp/prometheus-config-merger.go +++ b/otelcollector/shared/configmap/mp/prometheus-config-merger.go @@ -376,6 +376,7 @@ func populateDefaultPrometheusConfig() { if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_KUBESTATE_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType { kubestateMetricsKeepListRegex, exists := regexHash["KUBESTATE_METRICS_KEEP_LIST_REGEX"] kubestateScrapeInterval, intervalExists := intervalHash["KUBESTATE_SCRAPE_INTERVAL"] + log.Printf("path %s: %s\n", "kubeStateDefaultFile", kubeStateDefaultFile) if intervalExists { UpdateScrapeIntervalConfig(kubeStateDefaultFile, kubestateScrapeInterval) @@ -662,6 +663,30 @@ func populateDefaultPrometheusConfig() { } } + if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType { + acstorCapacityProvisionerKeepListRegex, exists := regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"] + acstorCapacityProvisionerScrapeInterval, intervalExists := intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"] + if intervalExists { + UpdateScrapeIntervalConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerScrapeInterval) + } + if exists && acstorCapacityProvisionerKeepListRegex != "" { + AppendMetricRelabelConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerKeepListRegex) + } + defaultConfigs = append(defaultConfigs, acstorCapacityProvisionerDefaultFile) + } + + if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType { + acstorMetricsExporterKeepListRegex, exists := regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"] + acstorMetricsExporterScrapeInterval, intervalExists := intervalHash["ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL"] + if intervalExists { + UpdateScrapeIntervalConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterScrapeInterval) + } + if exists && acstorMetricsExporterKeepListRegex != "" { + AppendMetricRelabelConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterKeepListRegex) + } + defaultConfigs = append(defaultConfigs, acstorMetricsExporterDefaultFile) + } + mergedDefaultConfigs = mergeDefaultScrapeConfigs(defaultConfigs) // if mergedDefaultConfigs != nil { // fmt.Printf("Merged default scrape targets: %v\n", mergedDefaultConfigs) @@ -1099,6 +1124,32 @@ func populateDefaultPrometheusConfigWithOperator() { } } + if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && (isConfigReaderSidecar() || currentControllerType == replicasetControllerType) { + acstorCapacityProvisionerKeepListRegex, exists := regexHash["ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX"] + acstorCapacityProvisionerScrapeInterval, intervalExists := intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"] + log.Printf("path %s: %s\n", "acstorCapacityProvisionerDefaultFile", acstorCapacityProvisionerDefaultFile) + if intervalExists { + UpdateScrapeIntervalConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerScrapeInterval) + } + if exists && acstorCapacityProvisionerKeepListRegex != "" { + AppendMetricRelabelConfig(acstorCapacityProvisionerDefaultFile, acstorCapacityProvisionerKeepListRegex) + } + defaultConfigs = append(defaultConfigs, acstorCapacityProvisionerDefaultFile) + } + + if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED"); exists && strings.ToLower(enabled) == "true" && (isConfigReaderSidecar() || currentControllerType == replicasetControllerType) { + acstorMetricsExporterKeepListRegex, exists := regexHash["ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX"] + acstorMetricsExporterScrapeInterval, intervalExists := intervalHash["ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL"] + log.Printf("path %s: %s\n", "acstorMetricsExporterDefaultFile", acstorMetricsExporterDefaultFile) + if intervalExists { + UpdateScrapeIntervalConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterScrapeInterval) + } + if exists && acstorMetricsExporterKeepListRegex != "" { + AppendMetricRelabelConfig(acstorMetricsExporterDefaultFile, acstorMetricsExporterKeepListRegex) + } + defaultConfigs = append(defaultConfigs, acstorMetricsExporterDefaultFile) + } + mergedDefaultConfigs = mergeDefaultScrapeConfigs(defaultConfigs) // if mergedDefaultConfigs != nil { // fmt.Printf("Merged default scrape targets: %v\n", mergedDefaultConfigs) @@ -1217,7 +1268,7 @@ func setDefaultFileScrapeInterval(scrapeInterval string) { prometheusCollectorHealthDefaultFile, windowsExporterDefaultRsSimpleFile, windowsExporterDefaultDsFile, windowsKubeProxyDefaultFileRsSimpleFile, windowsKubeProxyDefaultDsFile, podAnnotationsDefaultFile, kappieBasicDefaultFileDs, networkObservabilityRetinaDefaultFileDs, networkObservabilityHubbleDefaultFileDs, - networkObservabilityCiliumDefaultFileDs, + networkObservabilityCiliumDefaultFileDs, acstorMetricsExporterDefaultFile, acstorCapacityProvisionerDefaultFile, } for _, currentFile := range defaultFilesArray { diff --git a/otelcollector/shared/configmap/mp/tomlparser-default-scrape-settings.go b/otelcollector/shared/configmap/mp/tomlparser-default-scrape-settings.go index f2a52cb0..febf02bd 100644 --- a/otelcollector/shared/configmap/mp/tomlparser-default-scrape-settings.go +++ b/otelcollector/shared/configmap/mp/tomlparser-default-scrape-settings.go @@ -25,6 +25,8 @@ func (fcl *FilesystemConfigLoader) SetDefaultScrapeSettings() (map[string]string config["networkobservabilityHubble"] = "true" config["networkobservabilityCilium"] = "true" config["noDefaultsEnabled"] = "false" + config["acstor-capacity-provisioner"] = "true" + config["acstor-metrics-exporter"] = "true" return config, nil } @@ -46,6 +48,8 @@ func (fcl *FilesystemConfigLoader) ParseConfigMapForDefaultScrapeSettings() (map config["networkobservabilityHubble"] = "true" config["networkobservabilityCilium"] = "true" config["noDefaultsEnabled"] = "false" + config["acstor-capacity-provisioner"] = "true" + config["acstor-metrics-exporter"] = "true" if _, err := os.Stat(fcl.ConfigMapMountPath); os.IsNotExist(err) { fmt.Println("configmap for default scrape settings not mounted, using defaults") @@ -144,6 +148,16 @@ func (cp *ConfigProcessor) PopulateSettingValues(parsedConfig map[string]string) fmt.Printf("config::Using scrape settings for networkobservabilityCilium: %v\n", cp.NetworkObservabilityCilium) } + if val, ok := parsedConfig["acstor-capacity-provisioner"]; ok && val != "" { + cp.AcstorCapacityProvisioner = val + fmt.Printf("config:: Using scrape settings for acstor-capacity-provisioner: %v\n", cp.AcstorCapacityProvisioner) + } + + if val, ok := parsedConfig["acstor-metrics-exporter"]; ok && val != "" { + cp.AcstorMetricsExporter = val + fmt.Printf("config:: Using scrape settings for acstor-metrics-exporter: %v\n", cp.AcstorMetricsExporter) + } + if os.Getenv("MODE") == "" && strings.ToLower(strings.TrimSpace(os.Getenv("MODE"))) == "advanced" { controllerType := os.Getenv("CONTROLLER_TYPE") if controllerType == "ReplicaSet" && strings.ToLower(os.Getenv("OS_TYPE")) == "linux" && @@ -183,6 +197,8 @@ func (fcw *FileConfigWriter) WriteDefaultScrapeSettingsToFile(filename string, c file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED=%v\n", cp.NetworkObservabilityHubble)) file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED=%v\n", cp.NetworkObservabilityCilium)) file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_NO_DEFAULT_SCRAPING_ENABLED=%v\n", cp.NoDefaultsEnabled)) + file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_ACSTORCAPACITYPROVISIONER_SCRAPING_ENABLED=%v\n", cp.AcstorCapacityProvisioner)) + file.WriteString(fmt.Sprintf("AZMON_PROMETHEUS_ACSTORMETRICSEXPORTER_SCRAPING_ENABLED=%v\n", cp.AcstorMetricsExporter)) return nil } diff --git a/otelcollector/shared/configmap/mp/tomlparser-default-targets-metrics-keep-list.go b/otelcollector/shared/configmap/mp/tomlparser-default-targets-metrics-keep-list.go index 5a360cd5..b2ba91a5 100644 --- a/otelcollector/shared/configmap/mp/tomlparser-default-targets-metrics-keep-list.go +++ b/otelcollector/shared/configmap/mp/tomlparser-default-targets-metrics-keep-list.go @@ -18,6 +18,7 @@ var ( windowsExporterRegex, windowsKubeProxyRegex string networkobservabilityRetinaRegex, networkobservabilityHubbleRegex string networkobservabilityCiliumRegex, podAnnotationsRegex string + acstorCapacityProvisionerRegex, acstorMetricsExporterRegex string kubeletRegex_minimal_mac = "kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_used_bytes|kubelet_node_name|kubelet_running_pods|kubelet_running_pod_count|kubelet_running_sum_containers|kubelet_running_containers|kubelet_running_container_count|volume_manager_total_volumes|kubelet_node_config_error|kubelet_runtime_operations_total|kubelet_runtime_operations_errors_total|kubelet_runtime_operations_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_sum|kubelet_runtime_operations_duration_seconds_count|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_start_duration_seconds_sum|kubelet_pod_start_duration_seconds_count|kubelet_pod_worker_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_sum|kubelet_pod_worker_duration_seconds_count|storage_operation_duration_seconds_bucket|storage_operation_duration_seconds_sum|storage_operation_duration_seconds_count|storage_operation_errors_total|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_sum|kubelet_cgroup_manager_duration_seconds_count|kubelet_pleg_relist_interval_seconds_bucket|kubelet_pleg_relist_interval_seconds_count|kubelet_pleg_relist_interval_seconds_sum|kubelet_pleg_relist_duration_seconds_bucket|kubelet_pleg_relist_duration_seconds_count|kubelet_pleg_relist_duration_seconds_sum|rest_client_requests_total|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines|kubernetes_build_info|kubelet_certificate_manager_client_ttl_seconds|kubelet_certificate_manager_client_expiration_renew_errors|kubelet_server_expiration_renew_errors|kubelet_certificate_manager_server_ttl_seconds|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_inodes_free|kubelet_volume_stats_inodes_used|kubelet_volume_stats_inodes|kube_persistentvolumeclaim_access_mode|kube_persistentvolumeclaim_labels|kube_persistentvolume_status_phase" coreDNSRegex_minimal_mac = "coredns_build_info|coredns_panics_total|coredns_dns_responses_total|coredns_forward_responses_total|coredns_dns_request_duration_seconds|coredns_dns_request_duration_seconds_bucket|coredns_dns_request_duration_seconds_sum|coredns_dns_request_duration_seconds_count|coredns_forward_request_duration_seconds|coredns_forward_request_duration_seconds_bucket|coredns_forward_request_duration_seconds_sum|coredns_forward_request_duration_seconds_count|coredns_dns_requests_total|coredns_forward_requests_total|coredns_cache_hits_total|coredns_cache_misses_total|coredns_cache_entries|coredns_plugin_enabled|coredns_dns_request_size_bytes|coredns_dns_request_size_bytes_bucket|coredns_dns_request_size_bytes_sum|coredns_dns_request_size_bytes_count|coredns_dns_response_size_bytes|coredns_dns_response_size_bytes_bucket|coredns_dns_response_size_bytes_sum|coredns_dns_response_size_bytes_count|coredns_dns_response_size_bytes_bucket|coredns_dns_response_size_bytes_sum|coredns_dns_response_size_bytes_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines|kubernetes_build_info" cadvisorRegex_minimal_mac = "container_spec_cpu_quota|container_spec_cpu_period|container_memory_rss|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_packets_total|container_network_transmit_packets_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total|container_fs_reads_total|container_fs_writes_total|container_fs_reads_bytes_total|container_fs_writes_bytes_total|container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_cache|container_memory_swap|container_cpu_cfs_throttled_periods_total|container_cpu_cfs_periods_total|container_memory_rss|kubernetes_build_info|container_start_time_seconds" @@ -29,8 +30,10 @@ var ( networkobservabilityRetinaRegex_minimal_mac = "networkobservability.*" networkobservabilityHubbleRegex_minimal_mac = "hubble.*" // networkobservabilityCiliumRegex_minimal_mac = "cilium_drop.*|cilium_forward.*" - windowsexporterRegex_minimal_mac = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total" - windowskubeproxyRegex_minimal_mac = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines" + windowsexporterRegex_minimal_mac = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total" + windowskubeproxyRegex_minimal_mac = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines" + acstorCapacityProvisionerRegex_minimal_mac = "storage_pool_ready_state|storage_pool_capacity_used_bytes|storage_pool_capacity_provisioned_bytes|storage_pool_snapshot_capacity_reserved_bytes" + acstorMetricsExporter_minimal_mac = "disk_pool_ready_state|disk_read_operations_completed_total|disk_write_operations_completed_total|disk_read_operations_time_seconds_total|disk_write_operations_time_seconds_total|disk_errors_total|disk_read_bytes_total|disk_written_bytes_total" ) // getStringValue checks the type of the value and returns it as a string if possible. @@ -70,7 +73,7 @@ func parseConfigMapForKeepListRegex() map[string]interface{} { if minimalValue := getStringValue(tree.Get("minimalingestionprofile")); minimalValue != "" { configMap["minimalingestionprofile"] = minimalValue - } + } configMap["kubelet"] = getStringValue(tree.Get("kubelet")) configMap["coredns"] = getStringValue(tree.Get("coredns")) @@ -86,6 +89,8 @@ func parseConfigMapForKeepListRegex() map[string]interface{} { configMap["networkobservabilityHubble"] = getStringValue(tree.Get("networkobservabilityHubble")) configMap["networkobservabilityCilium"] = getStringValue(tree.Get("networkobservabilityCilium")) configMap["podannotations"] = getStringValue(tree.Get("podannotations")) + configMap["acstor-capacity-provisioner"] = getStringValue(tree.Get("acstor-capacity-provisioner")) + configMap["acstor-metrics-exporter"] = getStringValue(tree.Get("acstor-metrics-exporter")) fmt.Printf("Parsed config map for default-targets-metrics-keep-list: %v\n", configMap) @@ -101,21 +106,23 @@ func parseConfigMapForKeepListRegex() map[string]interface{} { func validateRegexValues(regexValues RegexValues) error { // Define a map of field names to their corresponding values fields := map[string]string{ - "kubelet": regexValues.kubelet, - "coredns": regexValues.coredns, - "cadvisor": regexValues.cadvisor, - "kubeproxy": regexValues.kubeproxy, - "apiserver": regexValues.apiserver, - "kubestate": regexValues.kubestate, - "nodeexporter": regexValues.nodeexporter, - "kappiebasic": regexValues.kappiebasic, - "windowsexporter": regexValues.windowsexporter, - "windowskubeproxy": regexValues.windowskubeproxy, - "networkobservabilityretina": regexValues.networkobservabilityretina, - "networkobservabilityhubble": regexValues.networkobservabilityhubble, - "networkobservabilitycilium": regexValues.networkobservabilitycilium, - "podannotations": regexValues.podannotations, - "minimalingestionprofile": regexValues.minimalingestionprofile, + "kubelet": regexValues.kubelet, + "coredns": regexValues.coredns, + "cadvisor": regexValues.cadvisor, + "kubeproxy": regexValues.kubeproxy, + "apiserver": regexValues.apiserver, + "kubestate": regexValues.kubestate, + "nodeexporter": regexValues.nodeexporter, + "kappiebasic": regexValues.kappiebasic, + "windowsexporter": regexValues.windowsexporter, + "windowskubeproxy": regexValues.windowskubeproxy, + "networkobservabilityretina": regexValues.networkobservabilityretina, + "networkobservabilityhubble": regexValues.networkobservabilityhubble, + "networkobservabilitycilium": regexValues.networkobservabilitycilium, + "podannotations": regexValues.podannotations, + "minimalingestionprofile": regexValues.minimalingestionprofile, + "acstor-capacity-provisioner": regexValues.acstorcapacityprovisioner, + "acstor-metrics-exporter": regexValues.acstormetricsexporter, } // Iterate over the fields and validate each regex @@ -145,6 +152,8 @@ func populateKeepListFromConfigMap(parsedConfig map[string]interface{}) (RegexVa networkobservabilitycilium: getStringValue(parsedConfig["networkobservabilityCilium"]), minimalingestionprofile: getStringValue(parsedConfig["minimalingestionprofile"]), podannotations: getStringValue(parsedConfig["podannotations"]), + acstorcapacityprovisioner: getStringValue(parsedConfig["acstor-capacity-provisioner"]), + acstormetricsexporter: getStringValue(parsedConfig["acstor-metrics-exporter"]), } // Validate regex values @@ -187,6 +196,8 @@ func populateRegexValuesWithMinimalIngestionProfile(regexValues RegexValues) { networkobservabilityHubbleRegex = fmt.Sprintf("%s|%s", regexValues.networkobservabilityhubble, networkobservabilityHubbleRegex_minimal_mac) networkobservabilityCiliumRegex = regexValues.networkobservabilitycilium podAnnotationsRegex = regexValues.podannotations + acstorCapacityProvisionerRegex = fmt.Sprintf("%s|%s", regexValues.acstorcapacityprovisioner, acstorCapacityProvisionerRegex_minimal_mac) + acstorMetricsExporterRegex = fmt.Sprintf("%s|%s", regexValues.acstormetricsexporter, acstorMetricsExporter_minimal_mac) // Print the updated regex strings after appending values // Only log this in debug mode @@ -221,6 +232,8 @@ func populateRegexValuesWithMinimalIngestionProfile(regexValues RegexValues) { networkobservabilityHubbleRegex = regexValues.networkobservabilityhubble networkobservabilityCiliumRegex = regexValues.networkobservabilitycilium podAnnotationsRegex = regexValues.podannotations + acstorCapacityProvisionerRegex = regexValues.acstorcapacityprovisioner + acstorMetricsExporterRegex = regexValues.acstormetricsexporter } } @@ -264,6 +277,8 @@ func tomlparserTargetsMetricsKeepList() { "NETWORKOBSERVABILITYRETINA_METRICS_KEEP_LIST_REGEX": networkobservabilityRetinaRegex, "NETWORKOBSERVABILITYHUBBLE_METRICS_KEEP_LIST_REGEX": networkobservabilityHubbleRegex, "NETWORKOBSERVABILITYCILIUM_METRICS_KEEP_LIST_REGEX": networkobservabilityCiliumRegex, + "ACSTORCAPACITYPROVISONER_KEEP_LIST_REGEX": acstorCapacityProvisionerRegex, + "ACSTORMETRICSEXPORTER_KEEP_LIST_REGEX": acstorMetricsExporterRegex, } out, err := yaml.Marshal(data) diff --git a/otelcollector/shared/configmap/mp/tomlparser-scrape-interval.go b/otelcollector/shared/configmap/mp/tomlparser-scrape-interval.go index 1b7493d6..baef42b9 100644 --- a/otelcollector/shared/configmap/mp/tomlparser-scrape-interval.go +++ b/otelcollector/shared/configmap/mp/tomlparser-scrape-interval.go @@ -14,7 +14,7 @@ import ( ) const ( - defaultScrapeInterval = "30s" + defaultScrapeInterval = "30s" ) var ( @@ -75,6 +75,8 @@ func processConfigMap() map[string]string { intervalHash["NETWORKOBSERVABILITYRETINA_SCRAPE_INTERVAL"] = checkDuration(getConfigStringValue(configMapSettings, "networkobservabilityRetina")) intervalHash["NETWORKOBSERVABILITYHUBBLE_SCRAPE_INTERVAL"] = checkDuration(getConfigStringValue(configMapSettings, "networkobservabilityHubble")) intervalHash["NETWORKOBSERVABILITYCILIUM_SCRAPE_INTERVAL"] = checkDuration(getConfigStringValue(configMapSettings, "networkobservabilityCilium")) + intervalHash["ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL"] = checkDuration(getConfigStringValue(configMapSettings, "acstor-capacity-provisioner")) + intervalHash["ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL"] = checkDuration(getConfigStringValue(configMapSettings, "acstor-metrics-exporter")) return intervalHash } else { @@ -93,7 +95,8 @@ func processConfigMap() map[string]string { "WINDOWSKUBEPROXY_SCRAPE_INTERVAL", "PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL", "POD_ANNOTATION_SCRAPE_INTERVAL", "KAPPIEBASIC_SCRAPE_INTERVAL", "NETWORKOBSERVABILITYRETINA_SCRAPE_INTERVAL", "NETWORKOBSERVABILITYHUBBLE_SCRAPE_INTERVAL", - "NETWORKOBSERVABILITYCILIUM_SCRAPE_INTERVAL", + "NETWORKOBSERVABILITYCILIUM_SCRAPE_INTERVAL", "ACSTORCAPACITYPROVISIONER_SCRAPE_INTERVAL", + "ACSTORMETRICSEXPORTER_SCRAPE_INTERVAL", } fmt.Printf("Setting default scrape interval (%s) for all jobs as no config map is present \n", defaultScrapeInterval) for _, key := range keys {