Skip to content

Commit

Permalink
Add dynamic_templates to app_metrics data stream (#6043)
Browse files Browse the repository at this point in the history
Add a "histogram" dynamic template to the application
metrics (`metrics-apm.app-*`) data stream index template,
so we can dynamically map histogram metrics when using
data streams.
  • Loading branch information
axw authored Sep 3, 2021
1 parent 31893d2 commit 43e491f
Show file tree
Hide file tree
Showing 13 changed files with 392 additions and 10 deletions.
3 changes: 3 additions & 0 deletions apmpackage/apm/changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
- description: define index sorting for internal metrics
type: enhancement
link: https://github.com/elastic/apm-server/pull/6116
- description: add histogram dynamic_template to app metrics data stream
type: enhancement
link: https://github.com/elastic/apm-server/pull/6043
- version: "0.4.0"
changes:
- description: add anonymous auth config, replace some RUM config
Expand Down
2 changes: 0 additions & 2 deletions apmpackage/apm/data_stream/app_metrics/fields/fields.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
- name: histogram
type: histogram
- name: kubernetes
title: Kubernetes
type: group
Expand Down
6 changes: 6 additions & 0 deletions apmpackage/apm/data_stream/app_metrics/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,9 @@ elasticsearch:
# as their names are application-specific and not
# known ahead of time.
dynamic: true
# Install dynamic templates for use in dynamically
# mapping complex application metrics.
dynamic_templates:
- histogram:
mapping:
type: histogram
2 changes: 0 additions & 2 deletions apmpackage/apm/data_stream/internal_metrics/fields/fields.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
- name: histogram
type: histogram
- name: kubernetes
title: Kubernetes
type: group
Expand Down
1 change: 0 additions & 1 deletion apmpackage/apm/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,6 @@ Metrics are written to `metrics-apm.app.*`, `metrics-apm.internal.*`, and `metri
|destination.ip|IP addess of the destination. Can be one of multiple IPv4 or IPv6 addresses.|ip| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
|destination.port|Port of the destination.|long| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
|ecs.version|ECS version the event conforms to.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
|histogram||histogram| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
|host.architecture|The architecture of the host the event was recorded on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
|host.hostname|The hostname of the host the event was recorded on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
|host.ip|IP of the host that records the event.|ip| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
Expand Down
1 change: 1 addition & 0 deletions apmpackage/cmd/gen-package/field.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type field struct {
// opt *in* to dynamically mapping where needed.
Dynamic bool `yaml:"dynamic,omitempty"`

DynamicTemplate bool `yaml:"dynamic_template,omitempty"`
ObjectTypeParams interface{} `yaml:"object_type_params,omitempty"`
Release string `yaml:"release,omitempty"`
Alias string `yaml:"alias,omitempty"`
Expand Down
7 changes: 7 additions & 0 deletions apmpackage/cmd/gen-package/genfields.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,13 @@ type fieldMapItem struct {
}

func (m fieldMap) update(f field) {
if f.DynamicTemplate {
// We don't add dynamic_template "fields" to the
// integration package; they are manually defined
// in the data stream manifest.
return
}

item := m[f.Name]
item.field = f
if item.fields == nil {
Expand Down
1 change: 1 addition & 0 deletions changelogs/head.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ https://github.com/elastic/apm-server/compare/7.13\...master[View commits]
- Data streams now define a default `dynamic` mapping parameter, overridable in the `<data-stream>@custom` template {pull}5947[5947]
- The `error.log.message` or `error.exception.message` field of errors will be copied to the ECS field `message` {pull}5974[5974]
- Define index sorting for internal metrics data stream {pull}6116[6116]
- Add histogram dynamic_template to app metrics data stream {pull}6043[6043]

[float]
==== Deprecated
Original file line number Diff line number Diff line change
@@ -0,0 +1,317 @@
{
"events": [
{
"@timestamp": "2017-05-30T18:53:41.364Z",
"agent": {
"name": "elastic-node",
"version": "3.14.0"
},
"data_stream.dataset": "apm.app.1234_service_12a3",
"data_stream.namespace": "default",
"data_stream.type": "metrics",
"ecs": {
"version": "dynamic"
},
"event": {
"agent_id_status": "missing",
"ingested": "dynamic"
},
"go.memstats.heap.sys.bytes": 6520832,
"host": {
"ip": "127.0.0.1"
},
"labels": {
"tag1": "one",
"tag2": 2
},
"metricset.name": "app",
"observer": {
"ephemeral_id": "dynamic",
"hostname": "dynamic",
"id": "dynamic",
"type": "apm-server",
"version": "dynamic",
"version_major": "dynamic"
},
"process": {
"pid": 1234
},
"processor": {
"event": "metric",
"name": "metric"
},
"service": {
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"user": {
"email": "user@mail.com",
"id": "axb123hg",
"name": "logged-in-user"
}
},
{
"@timestamp": "2017-05-30T18:53:41.366Z",
"agent": {
"name": "elastic-node",
"version": "3.14.0"
},
"data_stream.dataset": "apm.app.1234_service_12a3",
"data_stream.namespace": "default",
"data_stream.type": "metrics",
"ecs": {
"version": "dynamic"
},
"event": {
"agent_id_status": "missing",
"ingested": "dynamic"
},
"host": {
"ip": "127.0.0.1"
},
"labels": {
"tag1": "one",
"tag2": 2
},
"metricset.name": "app",
"observer": {
"ephemeral_id": "dynamic",
"hostname": "dynamic",
"id": "dynamic",
"type": "apm-server",
"version": "dynamic",
"version_major": "dynamic"
},
"process": {
"pid": 1234
},
"processor": {
"event": "metric",
"name": "metric"
},
"service": {
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"system.process.cgroup.memory.mem.limit.bytes": 2048,
"system.process.cgroup.memory.mem.usage.bytes": 1024,
"user": {
"email": "user@mail.com",
"id": "axb123hg",
"name": "logged-in-user"
}
},
{
"@timestamp": "2017-05-30T18:53:41.366Z",
"agent": {
"name": "elastic-node",
"version": "3.14.0"
},
"data_stream.dataset": "apm.app.1234_service_12a3",
"data_stream.namespace": "default",
"data_stream.type": "metrics",
"ecs": {
"version": "dynamic"
},
"event": {
"agent_id_status": "missing",
"ingested": "dynamic"
},
"host": {
"ip": "127.0.0.1"
},
"labels": {
"tag1": "one",
"tag2": 2
},
"metricset.name": "app",
"observer": {
"ephemeral_id": "dynamic",
"hostname": "dynamic",
"id": "dynamic",
"type": "apm-server",
"version": "dynamic",
"version_major": "dynamic"
},
"process": {
"pid": 1234
},
"processor": {
"event": "metric",
"name": "metric"
},
"service": {
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"system.process.cgroup.cpu.cfs.period.us": 1024,
"system.process.cgroup.cpu.cfs.quota.us": 2048,
"system.process.cgroup.cpu.id": 2048,
"system.process.cgroup.cpu.stats.periods": 2048,
"system.process.cgroup.cpu.stats.throttled.ns": 2048,
"system.process.cgroup.cpu.stats.throttled.periods": 2048,
"system.process.cgroup.cpuacct.id": 2048,
"system.process.cgroup.cpuacct.total.ns": 2048,
"user": {
"email": "user@mail.com",
"id": "axb123hg",
"name": "logged-in-user"
}
},
{
"@timestamp": "2017-05-30T18:53:41.366Z",
"agent": {
"name": "elastic-node",
"version": "3.14.0"
},
"data_stream.dataset": "apm.app.1234_service_12a3",
"data_stream.namespace": "default",
"data_stream.type": "metrics",
"ecs": {
"version": "dynamic"
},
"event": {
"agent_id_status": "missing",
"ingested": "dynamic"
},
"host": {
"ip": "127.0.0.1"
},
"labels": {
"tag1": "one",
"tag2": 2
},
"latency_distribution": {
"counts": [
1,
2,
3
],
"values": [
1.1,
2.2,
3.3
]
},
"metricset.name": "app",
"observer": {
"ephemeral_id": "dynamic",
"hostname": "dynamic",
"id": "dynamic",
"type": "apm-server",
"version": "dynamic",
"version_major": "dynamic"
},
"process": {
"pid": 1234
},
"processor": {
"event": "metric",
"name": "metric"
},
"service": {
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"user": {
"email": "user@mail.com",
"id": "axb123hg",
"name": "logged-in-user"
}
},
{
"@timestamp": "2017-05-30T18:53:42.281Z",
"agent": {
"name": "elastic-node",
"version": "3.14.0"
},
"data_stream.dataset": "apm.internal",
"data_stream.namespace": "default",
"data_stream.type": "metrics",
"ecs": {
"version": "dynamic"
},
"event": {
"agent_id_status": "missing",
"ingested": "dynamic"
},
"host": {
"ip": "127.0.0.1"
},
"labels": {
"code": 200,
"some": "abc",
"success": true,
"tag1": "one",
"tag2": 2
},
"metricset.name": "span_breakdown",
"observer": {
"ephemeral_id": "dynamic",
"hostname": "dynamic",
"id": "dynamic",
"type": "apm-server",
"version": "dynamic",
"version_major": "dynamic"
},
"process": {
"pid": 1234
},
"processor": {
"event": "metric",
"name": "metric"
},
"service": {
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"span": {
"self_time": {
"count": 1,
"sum.us": 633
},
"subtype": "mysql",
"type": "db"
},
"transaction": {
"breakdown.count": 12,
"duration": {
"count": 2,
"sum.us": 12
},
"name": "GET /",
"type": "request"
},
"user": {
"email": "user@mail.com",
"id": "axb123hg",
"name": "logged-in-user"
}
}
]
}
Loading

0 comments on commit 43e491f

Please sign in to comment.