diff --git a/ci/build.sh b/ci/build.sh index 2d8e58bc55..949c99115e 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -109,35 +109,20 @@ if [ -n "$GITHUB_TOKEN" ] && [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then # Generate override yaml files for chart dependencies to determine if changes are made to overrides yaml files echo "Generating overrides files..." - metrics_server_start=`grep -n "metrics-server:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - metrics_server_start=$(($metrics_server_start + 2)) - metrics_server_end=`grep -n "## Configure fluent-bit" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - metrics_server_end=$(($metrics_server_end - 1)) - - echo "Copy 'values.yaml' from line $metrics_server_start to line $metrics_server_end to 'metrics-server-overrides.yaml'" + echo "Copy metrics-server section from 'values.yaml' to 'metrics-server-overrides.yaml'" echo "# This file is auto-generated." > deploy/helm/metrics-server-overrides.yaml # Copy lines of metrics_server section and remove indention from values.yaml - sed -n "$metrics_server_start,${metrics_server_end}p" deploy/helm/sumologic/values.yaml | sed 's/ //' >> deploy/helm/metrics-server-overrides.yaml - - fluent_bit_start=`grep -n "fluent-bit:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - fluent_bit_start=$(($fluent_bit_start + 2)) - fluent_bit_end=`grep -n "## Configure prometheus-operator" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - fluent_bit_end=$(($fluent_bit_end - 1)) + yq r deploy/helm/sumologic/values.yaml metrics-server | yq d - enabled >> deploy/helm/metrics-server-overrides.yaml - echo "Copy 'values.yaml' from line $fluent_bit_start to line $fluent_bit_end to 'fluent-bit-overrides.yaml'" + echo "Copy fluent-bit section from 'values.yaml' to 'fluent-bit-overrides.yaml'" echo "# This file is auto-generated." > deploy/helm/fluent-bit-overrides.yaml # Copy lines of fluent-bit section and remove indention from values.yaml - sed -n "$fluent_bit_start,${fluent_bit_end}p" deploy/helm/sumologic/values.yaml | sed 's/ //' >> deploy/helm/fluent-bit-overrides.yaml - - prometheus_start=`grep -n "prometheus-operator:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - prometheus_start=$(($prometheus_start + 2)) - prometheus_end=`grep -n "## Configure falco" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - prometheus_end=$(($prometheus_end - 1)) + yq r deploy/helm/sumologic/values.yaml fluent-bit | yq d - enabled >> deploy/helm/fluent-bit-overrides.yaml - echo "Copy 'values.yaml' from line $prometheus_start to line $prometheus_end to 'prometheus-overrides.yaml'" + echo "Copy prometheus-operator section from 'values.yaml' to 'prometheus-overrides.yaml'" echo "# This file is auto-generated." > deploy/helm/prometheus-overrides.yaml # Copy lines of prometheus-operator section and remove indention from values.yaml - sed -n "$prometheus_start,${prometheus_end}p" deploy/helm/sumologic/values.yaml | sed 's/ //' >> deploy/helm/prometheus-overrides.yaml + yq r deploy/helm/sumologic/values.yaml prometheus-operator >> deploy/helm/prometheus-overrides.yaml echo "Copy prometheus.prometheusSpec.remoteWrite from 'prometheus-overrides.yaml' and inject into 'deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet'" prometheus_remote_write=`yq r deploy/helm/prometheus-overrides.yaml prometheus.prometheusSpec.remoteWrite -j | jq '.' | sed 's/^/ /'` @@ -148,16 +133,11 @@ if [ -n "$GITHUB_TOKEN" ] && [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then prometheus_remote_write="${prometheus_remote_write//$'\n'/\\n}" echo "// This file is autogenerated" > deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet sed "s#\[\/\*REMOTE_WRITE\*\/\]#$prometheus_remote_write#" ci/jsonnet-mixin.tmpl | sed 's#"http://collection-sumologic.sumologic.svc.cluster.local:9888\/#$._config.sumologicCollectorSvc + "#g' | sed 's/+: /+: /' | sed -r 's/"(\w*)":/\1:/g' > deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet - - falco_start=`grep -n "falco:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - falco_start=$(($falco_start + 2)) - falco_end=`grep -n "## Configure falco" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1` - falco_end=$(($falco_end - 1)) - echo "Copy 'values.yaml' from line $falco_start to end to 'falco-overrides.yaml'" + echo "Copy falco section from 'values.yaml' to 'falco-overrides.yaml'" echo "# This file is auto-generated." > deploy/helm/falco-overrides.yaml # Copy lines of falco section and remove indention from values.yaml - sed -n "$falco_start,$ p" deploy/helm/sumologic/values.yaml | sed 's/ //' >> deploy/helm/falco-overrides.yaml + yq r deploy/helm/sumologic/values.yaml falco | yq d - enabled >> deploy/helm/falco-overrides.yaml if [ "$(git diff deploy/helm/metrics-server-overrides.yaml)" ] || [ "$(git diff deploy/helm/fluent-bit-overrides.yaml)" ] || [ "$(git diff deploy/helm/prometheus-overrides.yaml)" ] || [ "$(git diff deploy/helm/falco-overrides.yaml)" ] || [ "$(git diff deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet)" ]; then echo "Detected changes in 'fluent-bit-overrides.yaml', 'prometheus-overrides.yaml', 'falco-overrides.yaml', or 'kube-prometheus-sumo-logic-mixin.libsonnet', committing the updated version to $TRAVIS_PULL_REQUEST_BRANCH..." diff --git a/deploy/helm/fluent-bit-overrides.yaml b/deploy/helm/fluent-bit-overrides.yaml index 08e25cc005..ea732ef8b8 100644 --- a/deploy/helm/fluent-bit-overrides.yaml +++ b/deploy/helm/fluent-bit-overrides.yaml @@ -13,24 +13,19 @@ backend: tls_verify: "on" tls_debug: 1 shared_key: - trackOffsets: true - tolerations: - - effect: NoSchedule - operator: Exists - +- effect: NoSchedule + operator: Exists input: systemd: enabled: true - parsers: enabled: true # This regex matches the first line of a multiline log starting with a date of the format : "2019-11-17 07:14:12" or "2019-11-17T07:14:12" regex: - - name: multi_line - regex: (?^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*) - + - name: multi_line + regex: (?^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*) rawConfig: |- @INCLUDE fluent-bit-service.conf @@ -101,4 +96,3 @@ rawConfig: |- Read_From_Tail true @INCLUDE fluent-bit-output.conf - diff --git a/deploy/helm/metrics-server-overrides.yaml b/deploy/helm/metrics-server-overrides.yaml index 372905e85e..e897c5ec35 100644 --- a/deploy/helm/metrics-server-overrides.yaml +++ b/deploy/helm/metrics-server-overrides.yaml @@ -1,5 +1,4 @@ # This file is auto-generated. args: - - --kubelet-insecure-tls - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - +- --kubelet-insecure-tls +- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname diff --git a/deploy/helm/prometheus-overrides.yaml b/deploy/helm/prometheus-overrides.yaml index 9242fe1d98..be17f95c83 100644 --- a/deploy/helm/prometheus-overrides.yaml +++ b/deploy/helm/prometheus-overrides.yaml @@ -1,4 +1,5 @@ # This file is auto-generated. +# Ensure we use pre 1.14 recording rules consistently as current content depends on them. kubeTargetVersionOverride: 1.13.0-0 enabled: true alertmanager: @@ -13,109 +14,108 @@ prometheusOperator: enabled: false prometheus: additionalServiceMonitors: - - name: collection-sumologic - additionalLabels: + - name: collection-sumologic + additionalLabels: + app: collection-sumologic + endpoints: + - port: metrics + namespaceSelector: + matchNames: + - sumologic + selector: + matchLabels: app: collection-sumologic - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic - - name: collection-sumologic-events - additionalLabels: + - name: collection-sumologic-events + additionalLabels: + app: collection-sumologic-events + endpoints: + - port: metrics + namespaceSelector: + matchNames: + - sumologic + selector: + matchLabels: app: collection-sumologic-events - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-events - - name: collection-fluent-bit - additionalLabels: - app: collection-fluent-bit - endpoints: - - port: metrics - path: /api/v1/metrics/prometheus - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: fluent-bit + - name: collection-fluent-bit + additionalLabels: + app: collection-fluent-bit + endpoints: + - port: metrics + path: /api/v1/metrics/prometheus + namespaceSelector: + matchNames: + - sumologic + selector: + matchLabels: + app: fluent-bit prometheusSpec: externalLabels: # Set this to a value to distinguish between different k8s clusters cluster: kubernetes remoteWrite: - # kube state metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.state - writeRelabelConfigs: - - action: keep - regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) - sourceLabels: [job, __name__] - # controller manager metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.controller-manager - writeRelabelConfigs: - - action: keep - regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* - sourceLabels: [job, __name__] - # scheduler metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.scheduler - writeRelabelConfigs: - - action: keep - regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* - sourceLabels: [job, __name__] - # api server metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.apiserver - writeRelabelConfigs: - - action: keep - regex: apiserver;(?:apiserver_request_count|apiserver_request_latencies.*|etcd_request_cache_get_latencies_summary.*|etcd_request_cache_add_latencies_summary.*|etcd_helper_cache_hit_count|etcd_helper_cache_miss_count) - sourceLabels: [job, __name__] - # kubelet metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.kubelet - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*) - sourceLabels: [job, __name__] - # cadvisor container metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: labelmap - regex: container_name - replacement: container - - action: drop - regex: POD - sourceLabels: [container] - - action: keep - regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|) - sourceLabels: [job,container,__name__] - # cadvisor aggregate container metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total) - sourceLabels: [job,__name__] - # node exporter metrics - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.node - writeRelabelConfigs: - - action: keep - regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files) - sourceLabels: [job, __name__] - # prometheus operator rules - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.operator.rule - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: [__name__] - # health - - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics - writeRelabelConfigs: - - action: keep - regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*) - sourceLabels: [__name__] - + - # kube state metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.state + writeRelabelConfigs: + - action: keep + regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) + sourceLabels: [job, __name__] + - # controller manager metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.controller-manager + writeRelabelConfigs: + - action: keep + regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* + sourceLabels: [job, __name__] + - # scheduler metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.scheduler + writeRelabelConfigs: + - action: keep + regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* + sourceLabels: [job, __name__] + - # api server metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.apiserver + writeRelabelConfigs: + - action: keep + regex: apiserver;(?:apiserver_request_count|apiserver_request_latencies.*|etcd_request_cache_get_latencies_summary.*|etcd_request_cache_add_latencies_summary.*|etcd_helper_cache_hit_count|etcd_helper_cache_miss_count) + sourceLabels: [job, __name__] + - # kubelet metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.kubelet + writeRelabelConfigs: + - action: keep + regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*) + sourceLabels: [job, __name__] + - # cadvisor container metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container + writeRelabelConfigs: + - action: labelmap + regex: container_name + replacement: container + - action: drop + regex: POD + sourceLabels: [container] + - action: keep + regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|) + sourceLabels: [job, container, __name__] + - # cadvisor aggregate container metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container + writeRelabelConfigs: + - action: keep + regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total) + sourceLabels: [job, __name__] + - # node exporter metrics + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.node + writeRelabelConfigs: + - action: keep + regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files) + sourceLabels: [job, __name__] + - # prometheus operator rules + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.operator.rule + writeRelabelConfigs: + - action: keep + regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' + sourceLabels: [__name__] + - # health + url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics + writeRelabelConfigs: + - action: keep + regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*) + sourceLabels: [__name__]