From 16c0834b6d3baae59ebcef874dddf78dd20f0c29 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 29 Jan 2019 16:47:08 +0100 Subject: [PATCH] [Metricbeat] Migrate php_fpm to ECS (#10366) This makes sure proper integration tests are run. --- CHANGELOG.next.asciidoc | 1 + dev-tools/ecs-migration.yml | 27 +++++++++ metricbeat/docs/fields.asciidoc | 20 +++++-- metricbeat/module/php_fpm/fields.go | 2 +- .../module/php_fpm/pool/_meta/data.json | 24 +++++--- .../php_fpm/pool/pool_integration_test.go | 17 ++++++ .../module/php_fpm/process/_meta/data.json | 58 ++++++++++++------- .../module/php_fpm/process/_meta/fields.yml | 20 +++++-- metricbeat/module/php_fpm/process/data.go | 49 +++++++++++----- .../process/process_integration_test.go | 20 ++++++- 10 files changed, 181 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5c2ad5f5587..29546291fe4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adjust Redis.info metricset fields to ECS. {pull}10319[10319] - Change type of field docker.container.ip_addresses to `ip` instead of `keyword. {pull}10364[10364] - Rename http.request.body field to http.request.body.content. {pull}10315[10315] +- Adjust php_fpm.process metricset fields to ECS. {pull}10366[10366] - Adjust mongodb.status metricset to to ECS. {pull}10368[10368] *Packetbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 73e5b61cea4..f44d4748798 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -1220,6 +1220,33 @@ ### Redis +- from: php_fpm.status.pid + to: process.pid + alias: true + beat: metricbeat + +- from: php_fpm.status.request_method + to: http.request.method + alias: true + beat: metricbeat + +- from: php_fpm.status.request_uri + to: url.original + alias: true + beat: metricbeat + +- from: php_fpm.status.content_length + to: http.response.body.bytes + alias: true + beat: metricbeat + +- from: php_fpm.status.user + to: http.response.user.name + alias: true + beat: metricbeat + +### Redis + - from: redis.info.server.version to: service.version alias: true diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 06e63df57a9..2fdd5cc13c5 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18855,7 +18855,9 @@ process contains the metrics that were obtained from the PHP-FPM process. *`php_fpm.process.pid`*:: + -- -type: integer +type: alias + +alias to: process.pid The PID of the process @@ -18917,7 +18919,9 @@ The duration in microseconds (1 million in a second) of the current request (my *`php_fpm.process.request_method`*:: + -- -type: keyword +type: alias + +alias to: http.request.method The request method (GET, POST, etc) (of the current request) @@ -18927,7 +18931,9 @@ The request method (GET, POST, etc) (of the current request) *`php_fpm.process.request_uri`*:: + -- -type: text +type: alias + +alias to: url.original The request URI with the query string (of the current request) @@ -18937,7 +18943,9 @@ The request URI with the query string (of the current request) *`php_fpm.process.content_length`*:: + -- -type: integer +type: alias + +alias to: http.response.body.bytes The content length of the request (only with POST) (of the current request) @@ -18947,7 +18955,9 @@ The content length of the request (only with POST) (of the current request) *`php_fpm.process.user`*:: + -- -type: keyword +type: alias + +alias to: user.name The user (PHP_AUTH_USER) (or - if not set) (for the current request) diff --git a/metricbeat/module/php_fpm/fields.go b/metricbeat/module/php_fpm/fields.go index 8fe2f02141e..fd226faf68c 100644 --- a/metricbeat/module/php_fpm/fields.go +++ b/metricbeat/module/php_fpm/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPhpFpm returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/php_fpm. func AssetPhpFpm() string { - return "eJzMWFtv47oRfs+vGOQlNuoY57y6QIHi9GyTh90a2exTUcg0NbKI8KIlR3a8v74YipJlWc5l4xarhwARzeE333xzoW7hCfcLqMoqKypzBUCKNC7genm3zD4tP19fAeQYpFcVKWcX8LcrAIDl3fL20/IzBPRb9BBIUB3AIHklA0inNUrCHArvTPvj+RVAKJ2nTDpbqM0CCqEDXgF41CgCLmAj+DdIpOwmLODf1yHo6xlcl0TV9X+uAAqFOg+LiOEWrDDYx84P7Ss25F1dpTcj8PlZpX0rkM6SUDYAldj5QKUg2KFHcGteHTiTXO6sVWKDIIXW8/Sqj/QIrXO6ezkG9wXIDWzn9Dsx829a3JV3EkOIOOY9y0O8fcz892ihxf2E+53z+WDtBfT8PJYYLYIrIrJjJL8YT0c2h5wd6/YtTCarmRFWbNBfjtSvJEjJGeR7K4yS4Dw4m6MRNp+PIpHOWpRsLoyiGFL9Bgx/dCZjdiCECqUqlIz/qkBKhvlg1xhZB5BCSqwIh1y0GLWzm5OlV0Am/dVmjZ4VqKx0RtkNePxeY6Ckjr4UUpErRegA/XXE7q5EC6JHLKjDBlDE/3o0btuXm1aB0ML3GmscctOyEBcvz4GsvUdLPS56mmhYKMUWYY1oQVlFShDmM1jXBNbRiNU9UufwHO45vVWArdA1svPW2dsf6B1zQftKcbncg0HRHie0HrHKNImtUFqsNbbBSImEAYTvXNF7WNdhPwNhc97mMa5aN2K1Z6CzTa4xH4+0+EytKObwIFRQI0xzoTFzI54zWSqde7QrmFTebVWOEUMLWAoLpbC5RlA05UZY6xxK1MMk4+cJsWrIS8HRbjeHR35ReVehpz0UTmu3CwcpFUJS5HHEYCvlRm4BtkqAgODkExJMHv9YcsEolEZYi4D5tKWwDqBsiV7RsErwE9yhistSeCEJfZPn/Loxf5LyrayZtAZPFhV+cYEb8axMbXoC73Jc2Qg6nssLFdp8PLz9nAjKSgTmkWtBIOFZ6Gfc67uWabQXdy+oHx37KZJDf/rox9tAlwaXagLLNF9cqAOoXF9eGAdBNMnZ6WG1E4pnT64EiZlV8mSi5jg/U/cO5Yft8X4RB8JpytmmAqaUL4TWsEbacVml8tS7VFKUzUIlPGYJ4yqmZFttBkvxhNC0oHGb7dyXJg+ux6s0LKzOSVhIUtv/B/1tK0ogDwzCbTO4KRszmUH/vhoxOwkOcIsWHLfgoubGwtJJJ8yiFY+h1gQ7FQPAsQSPIofVb6vpOQrIkThtShccPxjkXxLTh2x8qWb+j6JyWi2HoN5X//o9MfMoZHnhGeZLB5SUwdDEuJW5VkZRhBnHl3T+7NzYVhkgrzDEEYD9AuN4rkj4YbJz/imAs5ob73Bu56cycJPy6SZm6k07f99Mx0tv0G6XtR1ptPyO8PKWu9UxLyDatgf4jLKOkymvRHbwWSLmI5FZpU0Zw9Ruk/EWV9PqjDNMWhYFciFXvvSqhHQ2f5P8juEw5FE0uaDhQuG8EbQArJwss+bI93PPhpvRiQk+C3XQfT9yz20F/9GL7s/cazs31DFVjRPKEg7vuC9SyAQu7//RfRc4IacX3UH8zt+cXz2xae/HZ8LkPtc4g4faWmU3M0CS03EgYzI7I7JXJfYq1mN59QH3ZHYW52l2fiBQ9kx6vh3UaOm7CKLeVX4AhqeBF7Bkee0Fn3ApTK09HjCNkt61ZE1+B6O0TksikTjtblNpIGor98Tswe0s5FjEm7izY3JsvTBIpRtLyZ9KkBZDYxUm//zzcQbLf319bNICJuOYXwJYezWCjvCZfg7at4d72Ckq20ud30Mgz2PkO8BxBUVLfFfbUHkpBSSr0Fht49vFNU4UEToz+i4y6zDA8oEYsy2YLO+W2d+/Pd5l377++cBYPNyCKuKkHJCmMCmcfyu85sSLhNkIZZO9+I0d8w9i0yJQO3plsqpHUJ4MLG9A+QzCuNoSR9mgcX7ffOMT4ZDJ0tlQG/ag+SIo9E7sA/zGvvTLVfxeRlwc7uM9JnapNUpRB2yNS6FlrUX7rTF3FrsrYHdg707FJZDQG2UFYf4qMc0pv1oi/DcAAP//XeSCQA==" + return "eJzMWV1v67gRfc+vGOTlOqgj7L6mQIFiu9ubh90aublPRSGPqbFFhB9acmTH++uLoT4sy3ISN25RPVwgojk85/DMcKh7Dy+0f4CqrPJ1ZW8AWLOhB7hdfF3kvyx+vb0BKCiqoCvW3j3AX24AABZfF/e/LH6FSGFLASIj1xEscdAqgvLGkGIqYB287X6c3QDE0gfOlXdrvXmANZpINwCBDGGkB9ig/IaYtdvEB/jnbYzmdg63JXN1+68bgLUmU8SHhOEeHFoaYpeH95UECr6u2jcT8OVZtvOWoLxj1C4Cl9Rz4BIZdhQI/EpGR2Rayn20CjcECo3J2ldDpEdovTf9yym4b0BuYHtvLsQsv+lwV8ErijHhyAaRx3iHmOXfo4EO9wvtdz4Uo7E30MvzXFKKCH6dkB0j+T/T6SjmWLNj335EyTZqbtHhhsL1RP3GyFrNodg7tFqBD+BdQRZdkU0iUd45UhIuTqIYS/0BDD/1IVN2EMSKlF5rlf7UkbWK2WjWlFgHkKgUVUxjLTqMxrvNydA7IFv/1XZFQRyonfJWuw0E+r2myK07hlZoi1yJsQf054m4u5Ic4EBY0IcJoFn+DGT9dmg3oyOTg99rqmmsTadCGry+BqoOgRwPtBh4olGhxC3BisiBdpo1MhVzWNUMzvNE1D1xTziDR0lvHWGLpiYh77y7/4OCFy14X2kpl3uwhN1yaMxEVJEJt6gNrgx1m9EmEkXA0FMxe1jVcT8HdIVMC5RGnZ+IOgjQx2bfhE9LOnrlzhQZPKGOekJpKTQ2s/iaq1KbIpBbwqwKfqsLShg6wAodlOgKQ6D5Tg7C2hRQkhknmTwvRFUjXrs5xu8yeJYXVfAVBd7D2hvjd/FgpTUqTjpOBOys3NgtwlYjIESvXohh9vzTQgrGWhuCFUYq7joJ6wjalRQ0j6uEPNEfqrgqMaBiCk2ey+sm/EnKd7YW0Ro8eXL41Q1u8VXb2g4M3ue4dgl0WlcGKnLF9PYOcyJqpwhER6kFkTGI0c/QG1LLDbmr04v6j179difHfIbop4+BPg2udQgs2v7iSieALsz1jXEwRJOcvR+WO9TSe0olaJVZtkxmOqPsTN07lB+JJ/MxNYR3bc42FbBN+TUaAyvinZRVLk/ZtSVFuzxWGChvMS5TSnbVZjSUVojNETQds+v72s5D6vGybRaW5yyMivX2fyF/dxS1IA8Kwn3TuGmXMllA/7icCDuLHmhLDrwcwetaDhaxTrvCPEUJFGvDsNNpA2QvIRAWsPxheXdOAvaMp4fSFdsPAfmnVulDNr5VM/9Lu3JaLcegLqt/wzMxD4SqvHIP81sPlLWl2OxxZ3OjreYEM7Uv7frzc21bZYGDpphaAOEF1ktf0eKH2c6HlwjeGTl4x327PJWFL20+fUmZ+qXrv7/cTZfeaPwu706kyfI7octH7lbHugB2xx7QK6k6daYyktShV0VUTOzMsp2UC0zjN7lM8TUvz5AR0fJkkCtR+W1QJZR3xYfsdwxHIE+iKZDHA2sfLPIDUOVVmTdLXq69BG5aJxH4LNTR6fuZe25n+M9edP+Te21PQx9L1ZBAo3F0f0Yue9LZeJbVm4ANRQ6jdvBN6UX4xePf+u8JJ6IOXDHa9/M37ndXbNqC4zVh9lgYmsNT7Zx2mzkQq7tpIFP2PGPOd635LtZjWw4BD+x5FudpVjdAtWMaf8R4F4o7k9YfBzVZMq+CaPAJYARGuog3sORF3Vj3Wpi6eNKYWq2C78Sa/QhWG9MOYSviXX8LaxupruLP7B78zkFB63SD927Kjh0LS1z6S1K5ZK6y7o48MftTKd1xaOLC7O8/P89h8Y9vz01awWya81sE66AvYFcHk/mgN9qNWsCr0Pr+9Ag7zWV3EQ17iByk9b2AmFR9ciz3yw2Xl+9crLyLlK18sc9Wex5dAz/Fs4UGDbTOoL0xUyuV+MuWXrSbdRyl0zvbGClkJ5+wP8VNQsJs8XWR//X789f8+7efn4RBgHvQ63SxiMR3MFv78FFSzYoTtJhe+TJ0FrVr46X/kqDik9gMRu461VxV9QTKk/7uAyhfAa2vHYs3LFkf9s0nUYyHAqa8i7UVBs0HVDQ73Ef4QbgMq3T6vMhSEx/TtS8dzitSWEfqgis0qjbYfZotvKP+xtwvOLiCSuVnClY7ZCreFaZZ5VonwLXS598BAAD//9NS08Q=" } diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json index 6845096a33c..0fa62df1869 100644 --- a/metricbeat/module/php_fpm/pool/_meta/data.json +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -1,19 +1,21 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "event": { + "dataset": "php_fpm.pool", + "duration": 115000, + "module": "php_fpm" + }, "metricset": { - "host": "phpfpm:81", - "module": "php_fpm", - "name": "pool", - "rtt": 115 + "name": "pool" }, "php_fpm": { "pool": { "connections": { - "accepted": 1, + "accepted": 10, "listen_queue_len": 0, "max_listen_queue": 0, "queued": 0 @@ -28,8 +30,12 @@ "total": 2 }, "slow_requests": 0, - "start_since": 670, - "start_time": 1512631228 + "start_since": 600, + "start_time": 1548749474 } + }, + "service": { + "address": "127.0.0.1:81", + "type": "php_fpm" } -} +} \ No newline at end of file diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go index 24b4ae4c585..0fc2e43e10d 100644 --- a/metricbeat/module/php_fpm/pool/pool_integration_test.go +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -23,10 +23,27 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "phpfpm") + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("haproxy", "info").Fields.StringToPrint()) + +} func TestData(t *testing.T) { compose.EnsureUp(t, "phpfpm") f := mbtest.NewReportingMetricSetV2(t, getConfig()) diff --git a/metricbeat/module/php_fpm/process/_meta/data.json b/metricbeat/module/php_fpm/process/_meta/data.json index 8de0329a832..612fa24f87f 100644 --- a/metricbeat/module/php_fpm/process/_meta/data.json +++ b/metricbeat/module/php_fpm/process/_meta/data.json @@ -1,9 +1,25 @@ { - "@timestamp": "2018-09-25T16:47:04.998Z", + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "php_fpm.process", + "duration": 115000, + "module": "php_fpm" + }, + "http": { + "request": { + "method": "get" + }, + "response": { + "body": { + "bytes": 0 + } + } + }, "metricset": { - "module": "php_fpm", - "host": "phpfpm:81", - "rtt": 24681267, "name": "process" }, "php_fpm": { @@ -13,25 +29,25 @@ "process": { "last_request_cpu": 0, "last_request_memory": 2097152, - "request_duration": 1404, + "request_duration": 204, + "requests": 6, "script": "-", - "pid": 6, - "start_time": 1537974596, - "user": "-", - "state": "Idle", - "start_since": 3946, - "requests": 28, - "request_method": "GET", - "request_uri": "/status?json=", - "content_length": 0 + "start_since": 128, + "start_time": 1548769887, + "state": "Idle" } }, - "beat": { - "name": "host.example.com", - "hostname": "host.example.com", - "version": "7.0.0-alpha1" + "process": { + "pid": 17 + }, + "service": { + "address": "127.0.0.1:81", + "type": "php_fpm" + }, + "url": { + "original": "/status?full=\u0026json=" }, - "host": { - "name": "DESKTOP-RFOOE09" + "user": { + "name": "-" } -} +} \ No newline at end of file diff --git a/metricbeat/module/php_fpm/process/_meta/fields.yml b/metricbeat/module/php_fpm/process/_meta/fields.yml index 284b96fd148..252c75455c3 100644 --- a/metricbeat/module/php_fpm/process/_meta/fields.yml +++ b/metricbeat/module/php_fpm/process/_meta/fields.yml @@ -5,7 +5,9 @@ release: ga fields: - name: pid - type: integer + type: alias + path: process.pid + migration: true description: > The PID of the process - name: state @@ -30,19 +32,27 @@ description: > The duration in microseconds (1 million in a second) of the current request (my own definition) - name: request_method - type: keyword + type: alias + path: http.request.method + migration: true description: > The request method (GET, POST, etc) (of the current request) - name: request_uri - type: text + type: alias + path: url.original + migration: true description: > The request URI with the query string (of the current request) - name: content_length - type: integer + type: alias + path: http.response.body.bytes + migration: true description: > The content length of the request (only with POST) (of the current request) - name: user - type: keyword + type: alias + path: user.name + migration: true description: > The user (PHP_AUTH_USER) (or - if not set) (for the current request) - name: script diff --git a/metricbeat/module/php_fpm/process/data.go b/metricbeat/module/php_fpm/process/data.go index 4082180b710..bb8e6c5cacb 100644 --- a/metricbeat/module/php_fpm/process/data.go +++ b/metricbeat/module/php_fpm/process/data.go @@ -19,6 +19,7 @@ package process import ( "encoding/json" + "strings" "github.com/elastic/beats/metricbeat/mb" @@ -55,22 +56,40 @@ func eventsMapping(r mb.ReporterV2, content []byte) { } //remapping process details to match the naming format for _, process := range status.Processes { - proc := common.MapStr{ - "pid": process.PID, - "state": process.State, - "start_time": process.StartTime, - "start_since": process.StartSince, - "requests": process.Requests, - "request_duration": process.RequestDuration, - "request_method": process.RequestMethod, - "request_uri": process.RequestURI, - "content_length": process.ContentLength, - "user": process.User, - "script": process.Script, - "last_request_cpu": process.LastRequestCPU, - "last_request_memory": process.LastRequestMemory, + event := mb.Event{ + RootFields: common.MapStr{ + "http": common.MapStr{ + "request": common.MapStr{ + "method": strings.ToLower(process.RequestMethod), + }, + "response": common.MapStr{ + "body": common.MapStr{ + "bytes": process.ContentLength, + }, + }, + }, + "user": common.MapStr{ + "name": process.User, + }, + "process": common.MapStr{ + "pid": process.PID, + }, + "url": common.MapStr{ + "original": process.RequestURI, + }, + }, + MetricSetFields: common.MapStr{ + "state": process.State, + "start_time": process.StartTime, + "start_since": process.StartSince, + "requests": process.Requests, + "request_duration": process.RequestDuration, + "script": process.Script, + "last_request_cpu": process.LastRequestCPU, + "last_request_memory": process.LastRequestMemory, + }, } - event := mb.Event{MetricSetFields: proc} + event.ModuleFields = common.MapStr{} event.ModuleFields.Put("pool.name", status.Name) r.Event(event) diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go index 182f4815de4..6894b2c0e96 100644 --- a/metricbeat/module/php_fpm/process/process_integration_test.go +++ b/metricbeat/module/php_fpm/process/process_integration_test.go @@ -23,14 +23,32 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "phpfpm") + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("php_fpm", "process").Fields.StringToPrint()) + +} + func TestData(t *testing.T) { compose.EnsureUp(t, "phpfpm") f := mbtest.NewReportingMetricSetV2(t, getConfig()) - err := mbtest.WriteEventsReporterV2(f, t, "") + err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { t.Fatal("write", err) }