[Metricbeat] Migrate php_fpm to ECS (#10366)

This makes sure proper integration tests are run.
elastic · Jan 29, 2019 · 16c0834 · 16c0834
1 parent e9fe7ed
commit 16c0834
Show file tree

Hide file tree

Showing 10 changed files with 181 additions and 57 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adjust Redis.info metricset fields to ECS. {pull}10319[10319]
 - Change type of field docker.container.ip_addresses to `ip` instead of `keyword. {pull}10364[10364]
 - Rename http.request.body field to http.request.body.content. {pull}10315[10315]
+- Adjust php_fpm.process metricset fields to ECS. {pull}10366[10366]
 - Adjust mongodb.status metricset to to ECS. {pull}10368[10368]
 
 *Packetbeat*

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -1220,6 +1220,33 @@
 
 ### Redis
 
+- from: php_fpm.status.pid
+  to: process.pid
+  alias: true
+  beat: metricbeat
+
+- from: php_fpm.status.request_method
+  to: http.request.method
+  alias: true
+  beat: metricbeat
+
+- from: php_fpm.status.request_uri
+  to: url.original
+  alias: true
+  beat: metricbeat
+
+- from: php_fpm.status.content_length
+  to: http.response.body.bytes
+  alias: true
+  beat: metricbeat
+
+- from: php_fpm.status.user
+  to: http.response.user.name
+  alias: true
+  beat: metricbeat
+
+### Redis
+
 - from: redis.info.server.version
   to: service.version
   alias: true

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -18855,7 +18855,9 @@ process contains the metrics that were obtained from the PHP-FPM process.
 *`php_fpm.process.pid`*::
 +
 --
-type: integer
+type: alias
+
+alias to: process.pid
 
 The PID of the process
 
@@ -18917,7 +18919,9 @@ The duration in microseconds (1 million in a second) of the current request (my
 *`php_fpm.process.request_method`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: http.request.method
 
 The request method (GET, POST, etc) (of the current request)
 
@@ -18927,7 +18931,9 @@ The request method (GET, POST, etc) (of the current request)
 *`php_fpm.process.request_uri`*::
 +
 --
-type: text
+type: alias
+
+alias to: url.original
 
 The request URI with the query string (of the current request)
 
@@ -18937,7 +18943,9 @@ The request URI with the query string (of the current request)
 *`php_fpm.process.content_length`*::
 +
 --
-type: integer
+type: alias
+
+alias to: http.response.body.bytes
 
 The content length of the request (only with POST) (of the current request)
 
@@ -18947,7 +18955,9 @@ The content length of the request (only with POST) (of the current request)
 *`php_fpm.process.user`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: user.name
 
 The user (PHP_AUTH_USER) (or - if not set) (for the current request)
 

diff --git a/metricbeat/module/php_fpm/fields.go b/metricbeat/module/php_fpm/fields.go
diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json
@@ -1,19 +1,21 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
+    "agent": {
         "hostname": "host.example.com",
         "name": "host.example.com"
     },
+    "event": {
+        "dataset": "php_fpm.pool",
+        "duration": 115000,
+        "module": "php_fpm"
+    },
     "metricset": {
-        "host": "phpfpm:81",
-        "module": "php_fpm",
-        "name": "pool",
-        "rtt": 115
+        "name": "pool"
     },
     "php_fpm": {
         "pool": {
             "connections": {
-                "accepted": 1,
+                "accepted": 10,
                 "listen_queue_len": 0,
                 "max_listen_queue": 0,
                 "queued": 0
@@ -28,8 +30,12 @@
                 "total": 2
             },
             "slow_requests": 0,
-            "start_since": 670,
-            "start_time": 1512631228
+            "start_since": 600,
+            "start_time": 1548749474
         }
+    },
+    "service": {
+        "address": "127.0.0.1:81",
+        "type": "php_fpm"
     }
-}
+}
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -23,10 +23,27 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
+func TestFetch(t *testing.T) {
+	compose.EnsureUp(t, "phpfpm")
+
+	f := mbtest.NewReportingMetricSetV2(t, getConfig())
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("haproxy", "info").Fields.StringToPrint())
+
+}
 func TestData(t *testing.T) {
 	compose.EnsureUp(t, "phpfpm")
 	f := mbtest.NewReportingMetricSetV2(t, getConfig())

diff --git a/metricbeat/module/php_fpm/process/_meta/data.json b/metricbeat/module/php_fpm/process/_meta/data.json
@@ -1,9 +1,25 @@
 {
-    "@timestamp": "2018-09-25T16:47:04.998Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "agent": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "event": {
+        "dataset": "php_fpm.process",
+        "duration": 115000,
+        "module": "php_fpm"
+    },
+    "http": {
+        "request": {
+            "method": "get"
+        },
+        "response": {
+            "body": {
+                "bytes": 0
+            }
+        }
+    },
     "metricset": {
-        "module": "php_fpm",
-        "host": "phpfpm:81",
-        "rtt": 24681267,
         "name": "process"
     },
     "php_fpm": {
@@ -13,25 +29,25 @@
         "process": {
             "last_request_cpu": 0,
             "last_request_memory": 2097152,
-            "request_duration": 1404,
+            "request_duration": 204,
+            "requests": 6,
             "script": "-",
-            "pid": 6,
-            "start_time": 1537974596,
-            "user": "-",
-            "state": "Idle",
-            "start_since": 3946,
-            "requests": 28,
-            "request_method": "GET",
-            "request_uri": "/status?json=",
-            "content_length": 0
+            "start_since": 128,
+            "start_time": 1548769887,
+            "state": "Idle"
         }
     },
-    "beat": {
-        "name": "host.example.com",
-        "hostname": "host.example.com",
-        "version": "7.0.0-alpha1"
+    "process": {
+        "pid": 17
+    },
+    "service": {
+        "address": "127.0.0.1:81",
+        "type": "php_fpm"
+    },
+    "url": {
+        "original": "/status?full=\u0026json="
     },
-    "host": {
-        "name": "DESKTOP-RFOOE09"
+    "user": {
+        "name": "-"
     }
-}
+}
diff --git a/metricbeat/module/php_fpm/process/_meta/fields.yml b/metricbeat/module/php_fpm/process/_meta/fields.yml
@@ -5,7 +5,9 @@
   release: ga
   fields:
   - name: pid
-    type: integer
+    type: alias
+    path: process.pid
+    migration: true
     description: >
       The PID of the process
   - name: state
@@ -30,19 +32,27 @@
     description: >
       The duration in microseconds (1 million in a second) of the current request (my own definition)
   - name: request_method
-    type: keyword
+    type: alias
+    path: http.request.method
+    migration: true
     description: >
       The request method (GET, POST, etc) (of the current request)
   - name: request_uri
-    type: text
+    type: alias
+    path: url.original
+    migration: true
     description: >
       The request URI with the query string (of the current request)
   - name: content_length
-    type: integer
+    type: alias
+    path: http.response.body.bytes
+    migration: true
     description: >
       The content length of the request (only with POST) (of the current request)
   - name: user
-    type: keyword
+    type: alias
+    path: user.name
+    migration: true
     description: >
       The user (PHP_AUTH_USER) (or - if not set) (for the current request)
   - name: script

diff --git a/metricbeat/module/php_fpm/process/data.go b/metricbeat/module/php_fpm/process/data.go
@@ -19,6 +19,7 @@ package process
 
 import (
 	"encoding/json"
+	"strings"
 
 	"github.com/elastic/beats/metricbeat/mb"
 
@@ -55,22 +56,40 @@ func eventsMapping(r mb.ReporterV2, content []byte) {
 	}
 	//remapping process details to match the naming format
 	for _, process := range status.Processes {
-		proc := common.MapStr{
-			"pid":                 process.PID,
-			"state":               process.State,
-			"start_time":          process.StartTime,
-			"start_since":         process.StartSince,
-			"requests":            process.Requests,
-			"request_duration":    process.RequestDuration,
-			"request_method":      process.RequestMethod,
-			"request_uri":         process.RequestURI,
-			"content_length":      process.ContentLength,
-			"user":                process.User,
-			"script":              process.Script,
-			"last_request_cpu":    process.LastRequestCPU,
-			"last_request_memory": process.LastRequestMemory,
+		event := mb.Event{
+			RootFields: common.MapStr{
+				"http": common.MapStr{
+					"request": common.MapStr{
+						"method": strings.ToLower(process.RequestMethod),
+					},
+					"response": common.MapStr{
+						"body": common.MapStr{
+							"bytes": process.ContentLength,
+						},
+					},
+				},
+				"user": common.MapStr{
+					"name": process.User,
+				},
+				"process": common.MapStr{
+					"pid": process.PID,
+				},
+				"url": common.MapStr{
+					"original": process.RequestURI,
+				},
+			},
+			MetricSetFields: common.MapStr{
+				"state":               process.State,
+				"start_time":          process.StartTime,
+				"start_since":         process.StartSince,
+				"requests":            process.Requests,
+				"request_duration":    process.RequestDuration,
+				"script":              process.Script,
+				"last_request_cpu":    process.LastRequestCPU,
+				"last_request_memory": process.LastRequestMemory,
+			},
 		}
-		event := mb.Event{MetricSetFields: proc}
+
 		event.ModuleFields = common.MapStr{}
 		event.ModuleFields.Put("pool.name", status.Name)
 		r.Event(event)

diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go
@@ -23,14 +23,32 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
+func TestFetch(t *testing.T) {
+	compose.EnsureUp(t, "phpfpm")
+
+	f := mbtest.NewReportingMetricSetV2(t, getConfig())
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("php_fpm", "process").Fields.StringToPrint())
+
+}
+
 func TestData(t *testing.T) {
 	compose.EnsureUp(t, "phpfpm")
 	f := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(f, t, "")
+	err := mbtest.WriteEventsReporterV2(f, t, ".")
 	if err != nil {
 		t.Fatal("write", err)
 	}