[Metricbeat][vSphere] Support for configurable IntervalId for performance API

CHANGELOG.next.asciidoc

@@ -298,6 +298,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Disable event normalization for netflow input {pull}40635[40635]
 - Allow attribute selection in the Active Directory entity analytics provider. {issue}40482[40482] {pull}40662[40662]
 - Improve error quality when CEL program does not correctly return an events array. {pull}40580[40580]
+- Update support for period based intervalID in vSphere host and datastore metric sets {pull}40678[40678]
 
 *Auditbeat*
 

metricbeat/docs/modules/vsphere.asciidoc

@@ -9,14 +9,77 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-module-vsphere]]
 == vSphere module
 
-The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
+The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any VMware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0, and 6.5.
 
-By default it enables the metricsets `cluster`, `datastore`, `datastorecluster`, `host`, `network`, `resourcepool` and `virtualmachine`.
+By default, the vSphere module enables the following metric sets:
+
+1. cluster
+
+2. datastore
+
+3. datastorecluster
+
+4. host
+
+5. network
+
+6. resourcepool
+
+7. virtualmachine
+
+[float]
+=== Supported Periods:
+The Datastore and Host metric sets support performance data collection using the vSphere performance API. Since the performance API has usage restrictions based on data collection intervals, users should ensure that the period is configured optimally to receive real-time data. This configuration can be determined based on the https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html[Data Collection Intervals] and https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html[Data Collection Levels].
+
+[IMPORTANT]
+
+Only host and datastore metric sets have limitation of period restrictions.
+
+[float]
+==== Real-time data collection interval:
+- 20
+
+[float]
+==== Historical data collection interval:
+- 300
+- 1800
+- 7200
+- 86400
+
+[float]
+=== Example:
+If you need to configure multiple metric sets with different periods, you can achieve this by setting up multiple vSphere modules with different metric sets as demonstrated below:
+
+[source,yaml]
+----
+- module: vsphere
+  metricsets:
+   - cluster
+   - datastorecluster
+   - network
+   - resourcepool
+   - virtualmachine
+  period: 10s
+  hosts: ["https://localhost/sdk"]
+  username: "user"
+  password: "password"
+  insecure: false
+
+- module: vsphere
+  metricsets:
+   - datastore
+   - host
+  period: 300s
+  hosts: ["https://localhost/sdk"]
+  username: "user"
+  password: "password"
+  insecure: false
+----
 
 [float]
 === Dashboard
 
-The vsphere module comes with a predefined dashboard. For example:
+The vSphere module includes a predefined dashboard. For example:
 
 image::./images/metricbeat_vsphere_dashboard.png[]
 image::./images/metricbeat_vsphere_vm_dashboard.png[]
@@ -36,7 +99,16 @@ metricbeat.modules:
 - module: vsphere
   enabled: true
   metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
+
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
+  # Supported Periods:
+  # The Datastore and Host metric sets support performance data collection using the vSphere performance API.
+  # Since the performance API has usage restrictions based on data collection intervals,
+  # users should ensure that the period is configured optimally to receive real-time data.
+  # This configuration can be determined based on the Data Collection Intervals and Data Collection Levels.
+  # Reference Links:
+  # Data Collection Intervals: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html
+  # Data Collection Levels: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html
   period: 20s
   hosts: ["https://localhost/sdk"]
 

metricbeat/metricbeat.reference.yml

@@ -1008,7 +1008,16 @@ metricbeat.modules:
 - module: vsphere
   enabled: true
   metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
+
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
+  # Supported Periods:
+  # The Datastore and Host metric sets support performance data collection using the vSphere performance API.
+  # Since the performance API has usage restrictions based on data collection intervals,
+  # users should ensure that the period is configured optimally to receive real-time data.
+  # This configuration can be determined based on the Data Collection Intervals and Data Collection Levels.
+  # Reference Links:
+  # Data Collection Intervals: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html
+  # Data Collection Levels: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html
   period: 20s
   hosts: ["https://localhost/sdk"]
 

metricbeat/module/vsphere/_meta/config.reference.yml

@@ -1,7 +1,16 @@
 - module: vsphere
   enabled: true
   metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
+
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
+  # Supported Periods:
+  # The Datastore and Host metric sets support performance data collection using the vSphere performance API.
+  # Since the performance API has usage restrictions based on data collection intervals,
+  # users should ensure that the period is configured optimally to receive real-time data.
+  # This configuration can be determined based on the Data Collection Intervals and Data Collection Levels.
+  # Reference Links:
+  # Data Collection Intervals: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html
+  # Data Collection Levels: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html
   period: 20s
   hosts: ["https://localhost/sdk"]
 

metricbeat/module/vsphere/_meta/config.yml

@@ -7,7 +7,16 @@
   #  - network
   #  - resourcepool
   #  - virtualmachine
+
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
+  # Supported Periods:
+  # The Datastore and Host metric sets support performance data collection using the vSphere performance API.
+  # Since the performance API has usage restrictions based on data collection intervals,
+  # users should ensure that the period is configured optimally to receive real-time data.
+  # This configuration can be determined based on the Data Collection Intervals and Data Collection Levels.
+  # Reference Links:
+  # Data Collection Intervals: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html
+  # Data Collection Levels: https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html
   period: 20s
   hosts: ["https://localhost/sdk"]
 

metricbeat/module/vsphere/_meta/docs.asciidoc

@@ -1,11 +1,74 @@
-The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
+The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any VMware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0, and 6.5.
 
-By default it enables the metricsets `cluster`, `datastore`, `datastorecluster`, `host`, `network`, `resourcepool` and `virtualmachine`.
+By default, the vSphere module enables the following metric sets:
+
+1. cluster
+
+2. datastore
+
+3. datastorecluster
+
+4. host
+
+5. network
+
+6. resourcepool
+
+7. virtualmachine
+
+[float]
+=== Supported Periods:
+The Datastore and Host metric sets support performance data collection using the vSphere performance API. Since the performance API has usage restrictions based on data collection intervals, users should ensure that the period is configured optimally to receive real-time data. This configuration can be determined based on the https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html[Data Collection Intervals] and https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html[Data Collection Levels].
+
+[IMPORTANT]
+
+Only host and datastore metric sets have limitation of period restrictions.
+
+[float]
+==== Real-time data collection interval:
+- 20
+
+[float]
+==== Historical data collection interval:
+- 300
+- 1800
+- 7200
+- 86400
+
+[float]
+=== Example:
+If you need to configure multiple metric sets with different periods, you can achieve this by setting up multiple vSphere modules with different metric sets as demonstrated below:
+
+[source,yaml]
+----
+- module: vsphere
+  metricsets:
+   - cluster
+   - datastorecluster
+   - network
+   - resourcepool
+   - virtualmachine
+  period: 10s
+  hosts: ["https://localhost/sdk"]
+  username: "user"
+  password: "password"
+  insecure: false
+
+- module: vsphere
+  metricsets:
+   - datastore
+   - host
+  period: 300s
+  hosts: ["https://localhost/sdk"]
+  username: "user"
+  password: "password"
+  insecure: false
+----
 
 [float]
 === Dashboard
 
-The vsphere module comes with a predefined dashboard. For example:
+The vSphere module includes a predefined dashboard. For example:
 
 image::./images/metricbeat_vsphere_dashboard.png[]
 image::./images/metricbeat_vsphere_vm_dashboard.png[]

metricbeat/module/vsphere/datastore/datastore.go

@@ -80,6 +80,12 @@ func (m *DataStoreMetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	period := m.Module().Config().Period
+	if !isValidPeriod(period.Seconds()) {
+		m.Logger().Errorf("Invalid period %v. Please provide one of the following values: 20, 300, 1800, 7200, 86400", period)
+		return nil
+	}
+
 	client, err := govmomi.NewClient(ctx, m.HostURL, m.Insecure)
 	if err != nil {
 		return fmt.Errorf("error in NewClient: %w", err)
@@ -144,38 +150,9 @@ func (m *DataStoreMetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2)
 				continue
 			}
 
-			spec := types.PerfQuerySpec{
-				Entity:     dst[i].Reference(),
-				MetricId:   metricIds,
-				MaxSample:  1,
-				IntervalId: 20, // right now we are only grabbing real time metrics from the performance manager
-			}
-
-			// Query performance data
-			samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec})
-			if err != nil {
-				m.Logger().Debugf("Failed to query performance data for host %s: %v", dst[i].Name, err)
-				continue
-			}
-
-			if len(samples) == 0 {
-				m.Logger().Debugf("No samples returned from performance manager")
-				continue
-			}
-
-			results, err := perfManager.ToMetricSeries(ctx, samples)
+			metricMap, err := m.getPerfMetrics(ctx, perfManager, dst[i], metricIds)
 			if err != nil {
-				m.Logger().Debugf("Failed to query performance data to metric series for host %s: %v", dst[i].Name, err)
-				continue
-			}
-
-			metricMap := make(map[string]interface{})
-			for _, result := range results[0].Value {
-				if len(result.Value) > 0 {
-					metricMap[result.Name] = result.Value[0]
-					continue
-				}
-				m.Logger().Debugf("For host %s,Metric %v: No result found", dst[i].Name, result.Name)
+				m.Logger().Errorf("Failed to retrieve performance metrics from host %s: %w", dst[i].Name, err)
 			}
 
 			reporter.Event(mb.Event{
@@ -191,7 +168,6 @@ func (m *DataStoreMetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2)
 }
 
 func getAssetNames(ctx context.Context, pc *property.Collector, ds *mo.Datastore) (*assetNames, error) {
-
 	outputVmNames := make([]string, 0, len(ds.Vm))
 	if len(ds.Vm) > 0 {
 		var objects []mo.ManagedEntity
@@ -235,3 +211,62 @@ func getAssetNames(ctx context.Context, pc *property.Collector, ds *mo.Datastore
 		outputVmNames:   outputVmNames,
 	}, nil
 }
+
+func (m *DataStoreMetricSet) getPerfMetrics(ctx context.Context, perfManager *performance.Manager, dst mo.Datastore, metricIds []types.PerfMetricId) (map[string]interface{}, error) {
+	metricMap := make(map[string]interface{})
+	summary, err := perfManager.ProviderSummary(ctx, dst.Reference())
+	if err != nil {
+		return metricMap, fmt.Errorf("failed to get summary: %w", err)
+	}
+
+	var refreshRate = int32(m.Module().Config().Period.Seconds())
+	if summary.CurrentSupported {
+		refreshRate = summary.RefreshRate
+		if int32(m.Module().Config().Period.Seconds()) != refreshRate {
+			m.Logger().Warnf("User-provided period %v does not match system's refresh rate %v. Risk of data duplication. Consider adjusting period.", m.Module().Config().Period, refreshRate)
+		}
+	} else {
+		m.Logger().Warnf("Live data collection not supported. Use one of the system's historical interval (300, 1800, 7200, 86400). Risk of data duplication. Consider adjusting period.")
+	}
+
+	spec := types.PerfQuerySpec{
+		Entity:     dst.Reference(),
+		MetricId:   metricIds,
+		MaxSample:  1,
+		IntervalId: refreshRate, // using refreshRate as interval
+	}
+
+	// Query performance data
+	samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec})
+	if err != nil {
+		return metricMap, fmt.Errorf("failed to query performance data: %w", err)
+	}
+
+	if len(samples) == 0 {
+		m.Logger().Debug("No samples returned from performance manager")
+		return metricMap, nil
+	}
+
+	results, err := perfManager.ToMetricSeries(ctx, samples)
+	if err != nil {
+		m.Logger().Errorf("failed to convert performance data to metric series: %v", err)
+	}
+
+	for _, result := range results[0].Value {
+		if len(result.Value) > 0 {
+			metricMap[result.Name] = result.Value[0]
+			continue
+		}
+		m.Logger().Debugf("For datastore %s, Metric %v: No result found", dst.Name, result.Name)
+	}
+
+	return metricMap, nil
+}
+
+func isValidPeriod(period float64) bool {
+	switch period {
+	case 20, 300, 1800, 7200, 86400:
+		return true
+	}
+	return false
+}