Merge branch 'main' of https://github.com/niraj-elastic/beats into vs…

…phere_add_alerts_metrics

Conflicts:
	metricbeat/module/vsphere/datastore/datastore.go
	metricbeat/module/vsphere/fields.go
	metricbeat/module/vsphere/host/host.go
	metricbeat/module/vsphere/virtualmachine/virtualmachine.go

.github/workflows/pre-commit.yml

@@ -15,4 +15,4 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: elastic/apm-pipeline-library/.github/actions/pre-commit@current
+      - uses: elastic/oblt-actions/pre-commit@v1

.github/workflows/updatecli.d/values.d/ironbank.yml

@@ -4,4 +4,3 @@ config:
   - path: dev-tools/packaging/templates/ironbank/heartbeat
   - path: dev-tools/packaging/templates/ironbank/metricbeat
   - path: dev-tools/packaging/templates/ironbank/packetbeat
-  - beats_packages: dev-tools/packaging/packages.yml

.mergify.yml

@@ -1,5 +1,6 @@
 queue_rules:
   - name: default
+    merge_method: squash
     conditions:
       - check-success=buildkite/beats
 pull_request_rules:
@@ -72,7 +73,6 @@ pull_request_rules:
       - "#approved-reviews-by>=1"
     actions:
       queue:
-        method: squash
         name: default
   - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
@@ -116,7 +116,6 @@ pull_request_rules:
       - "#approved-reviews-by>=1"
     actions:
       queue:
-        method: squash
         name: default
   - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
     conditions:
@@ -148,16 +147,17 @@ pull_request_rules:
           branches, such as:
           * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit
 
-  - name: add backport-8.x label for main only
+  - name: add backport-8.x label for main only if no skipped or assigned already
     conditions:
-      - -label~=^backport-8.x
+      - -label~=^(backport-skip|backport-8.x)$
       - base=main
       - -merged
       - -closed
     actions:
       comment:
         message: |
           `backport-8.x` has been added to help with the transition to the new branch `8.x`.
+          If you don't need it please use `backport-skip` label and remove the `backport-8.x` label.
       label:
         add:
           - backport-8.x

CHANGELOG.next.asciidoc

@@ -163,6 +163,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix credential handling when workload identity is being used in GCS input. {issue}39977[39977] {pull}40663[40663]
 - Fix publication of group data from the Okta entity analytics provider. {pull}40681[40681]
 - Ensure netflow custom field configuration is applied. {issue}40735[40735] {pull}40730[40730]
+- Fix replace processor handling of zero string replacement validation. {pull}40751[40751]
 
 *Heartbeat*
 
@@ -334,6 +335,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Update fields to use mapstr in vSphere virtualmachine metricset  {pull}40707[40707]
 - Add metrics related to alert in all the vSphere metricsets. {pull}40714[40714]
 - Add support for period based intervalID in vSphere host and datastore metricsets {pull}40678[40678]
+- Add new metrics fot datastore and minor changes to overall vSphere metrics {pull}40766[40766]
+- Add `metrics_count` to Prometheus module if `metrics_count: true` is set. {pull}40411[40411]
 
 *Metricbeat*
 

libbeat/processors/actions/replace.go

@@ -45,7 +45,14 @@ type replaceStringConfig struct {
 type replaceConfig struct {
 	Field       string         `config:"field" validate:"required"`
 	Pattern     *regexp.Regexp `config:"pattern" validate:"required"`
-	Replacement string         `config:"replacement" validate:"required"`
+	Replacement *string        `config:"replacement"`
+}
+
+func (c replaceConfig) Validate() error {
+	if c.Replacement == nil {
+		return errors.New("missing replacement")
+	}
+	return nil
 }
 
 func init() {
@@ -82,7 +89,7 @@ func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) {
 	}
 
 	for _, field := range f.config.Fields {
-		err := f.replaceField(field.Field, field.Pattern, field.Replacement, event)
+		err := f.replaceField(field.Field, field.Pattern, *field.Replacement, event)
 		if err != nil {
 			errMsg := fmt.Errorf("Failed to replace fields in processor: %w", err)
 			f.log.Debugw(errMsg.Error(), logp.TypeKey, logp.EventType)

libbeat/processors/actions/replace_test.go

@@ -45,7 +45,7 @@ func TestBadConfig(t *testing.T) {
 		},
 		{
 			name:        "no-regex",
-			cfg:         replaceStringConfig{Fields: []replaceConfig{{Field: "message", Replacement: "new_message"}}},
+			cfg:         replaceStringConfig{Fields: []replaceConfig{{Field: "message", Replacement: ptr("new_message")}}},
 			shouldError: true,
 		},
 		{
@@ -56,15 +56,20 @@ func TestBadConfig(t *testing.T) {
 		{
 			name: "valid-then-invalid",
 			cfg: replaceStringConfig{Fields: []replaceConfig{
-				{Field: "message", Pattern: regexp.MustCompile(`message`), Replacement: "new_message"},
+				{Field: "message", Pattern: regexp.MustCompile(`message`), Replacement: ptr("new_message")},
 				{Field: "message", Pattern: regexp.MustCompile(`message`)},
 			},
 			},
 			shouldError: true,
 		},
 		{
 			name:        "no-error",
-			cfg:         replaceStringConfig{Fields: []replaceConfig{{Field: "message", Replacement: "new_message", Pattern: regexp.MustCompile(`message`)}}},
+			cfg:         replaceStringConfig{Fields: []replaceConfig{{Field: "message", Replacement: ptr("new_message"), Pattern: regexp.MustCompile(`message`)}}},
+			shouldError: false,
+		},
+		{
+			name:        "no-error zero string",
+			cfg:         replaceStringConfig{Fields: []replaceConfig{{Field: "message", Replacement: ptr(""), Pattern: regexp.MustCompile(`message`)}}},
 			shouldError: false,
 		},
 	}
@@ -102,7 +107,7 @@ func TestReplaceRun(t *testing.T) {
 				{
 					Field:       "f",
 					Pattern:     regexp.MustCompile(`a`),
-					Replacement: "b",
+					Replacement: ptr("b"),
 				},
 			},
 			Input: mapstr.M{
@@ -115,13 +120,32 @@ func TestReplaceRun(t *testing.T) {
 			IgnoreMissing: false,
 			FailOnError:   true,
 		},
+		{
+			description: "replace with zero",
+			Fields: []replaceConfig{
+				{
+					Field:       "f",
+					Pattern:     regexp.MustCompile(`a`),
+					Replacement: ptr(""),
+				},
+			},
+			Input: mapstr.M{
+				"f": "abc",
+			},
+			Output: mapstr.M{
+				"f": "bc",
+			},
+			error:         false,
+			IgnoreMissing: false,
+			FailOnError:   true,
+		},
 		{
 			description: "Add one more hierarchy to event",
 			Fields: []replaceConfig{
 				{
 					Field:       "f.b",
 					Pattern:     regexp.MustCompile(`a`),
-					Replacement: "b",
+					Replacement: ptr("b"),
 				},
 			},
 			Input: mapstr.M{
@@ -144,12 +168,12 @@ func TestReplaceRun(t *testing.T) {
 				{
 					Field:       "f",
 					Pattern:     regexp.MustCompile(`a.*c`),
-					Replacement: "cab",
+					Replacement: ptr("cab"),
 				},
 				{
 					Field:       "g",
 					Pattern:     regexp.MustCompile(`ef`),
-					Replacement: "oor",
+					Replacement: ptr("oor"),
 				},
 			},
 			Input: mapstr.M{
@@ -170,12 +194,12 @@ func TestReplaceRun(t *testing.T) {
 				{
 					Field:       "f",
 					Pattern:     regexp.MustCompile(`abc`),
-					Replacement: "xyz",
+					Replacement: ptr("xyz"),
 				},
 				{
 					Field:       "g",
 					Pattern:     regexp.MustCompile(`def`),
-					Replacement: "",
+					Replacement: nil,
 				},
 			},
 			Input: mapstr.M{
@@ -216,11 +240,13 @@ func TestReplaceRun(t *testing.T) {
 				assert.Error(t, err)
 			}
 
-			assert.True(t, reflect.DeepEqual(newEvent.Fields, test.Output))
+			assert.Equal(t, newEvent.Fields, test.Output)
 		})
 	}
 }
 
+func ptr[T any](v T) *T { return &v }
+
 func TestReplaceField(t *testing.T) {
 	var tests = []struct {
 		Field         string
@@ -322,7 +348,7 @@ func TestReplaceField(t *testing.T) {
 					{
 						Field:       "@metadata.f",
 						Pattern:     regexp.MustCompile(`a`),
-						Replacement: "b",
+						Replacement: ptr("b"),
 					},
 				},
 			},

libbeat/tests/system/requirements.txt

@@ -13,7 +13,7 @@ certifi==2024.7.4
 cffi==1.16.0
 chardet==3.0.4
 charset-normalizer==3.3.2
-cryptography==42.0.4
+cryptography==43.0.1
 deepdiff==4.2.0
 Deprecated==1.2.14
 distro==1.9.0

metricbeat/docs/fields.asciidoc

@@ -58258,6 +58258,16 @@ Stats scraped from a Prometheus endpoint.
 
 
 
+*`metrics_count`*::
++
+--
+Number of metrics per Elasticsearch document.
+
+
+type: long
+
+--
+
 
 *`prometheus.labels.*`*::
 +
@@ -67020,76 +67030,92 @@ format: percent
 
 --
 
-*`vsphere.datastore.fstype`*::
+*`vsphere.datastore.disk.capacity.bytes`*::
 +
 --
-Filesystem type.
+Configured size of the datastore.
 
 
-type: keyword
+type: long
+
+format: bytes
 
 --
 
-*`vsphere.datastore.host.count`*::
+*`vsphere.datastore.disk.capacity.usage.bytes`*::
 +
 --
-Number of hosts.
+The amount of storage capacity currently being consumed by datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
-*`vsphere.datastore.host.names`*::
+*`vsphere.datastore.disk.provisioned.bytes`*::
 +
 --
-List of all the host names.
+Amount of storage set aside for use by a datastore.
+
+
+type: long
+
+format: bytes
+
+--
+
+*`vsphere.datastore.fstype`*::
++
+--
+Filesystem type.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.iops`*::
+*`vsphere.datastore.host.count`*::
 +
 --
-Storage I/O Control aggregated Input/Output Operations Per Second.
+Number of hosts.
 
 
 type: long
 
 --
 
-*`vsphere.datastore.name`*::
+*`vsphere.datastore.host.names`*::
 +
 --
-Datastore name.
+List of all the host names.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.read.bytes`*::
+*`vsphere.datastore.name`*::
 +
 --
-Rate of reading data from the datastore.
-
+Datastore name.
 
-type: long
 
-format: bytes
+type: keyword
 
 --
 
-*`vsphere.datastore.read.latency.total.ms`*::
+*`vsphere.datastore.read.bytes`*::
 +
 --
-Average amount of time for a read operation from the datastore in milliseconds.
+Rate of reading data from the datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
 *`vsphere.datastore.status`*::
@@ -67144,16 +67170,6 @@ format: bytes
 
 --
 
-*`vsphere.datastore.write.latency.total.ms`*::
-+
---
-Average amount of time for a write operation from the datastore in milliseconds.
-
-
-type: long
-
---
-
 [float]
 === datastorecluster
 
@@ -68176,10 +68192,10 @@ type: long
 --
 
 
-*`vsphere.virtualmachine.snapshot.info`*::
+*`vsphere.virtualmachine.snapshot.info.*`*::
 +
 --
-Deatils of the snapshots of this virtualmachine.
+Details of the snapshots of this virtualmachine.
 
 type: object