Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove metricIDNameMap and use ListMetrics to construct metric queries #10627

Merged
merged 84 commits into from
Feb 19, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
84 commits
Select commit Hold shift + click to select a range
aed6cc6
Never default to a qualifier when none of them are set. (#9148)
ph Nov 19, 2018
07c4fa5
Merge branch 'master' of github.com:kaiyan-sheng/beats
kaiyan-sheng Nov 27, 2018
ff52365
Merge branch 'master' of github.com:kaiyan-sheng/beats
kaiyan-sheng Dec 10, 2018
5f9f575
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 10, 2018
09d8b97
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 10, 2018
141277e
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 13, 2018
276a1de
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 14, 2018
19a705a
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 17, 2018
e54a566
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 18, 2018
edd4488
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 26, 2018
b0d2e22
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 27, 2018
378b4f9
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 28, 2018
1996739
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Dec 28, 2018
1e05766
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 2, 2019
33c130c
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 2, 2019
513bbd0
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Jan 3, 2019
c618fd0
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Jan 3, 2019
40c8368
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 3, 2019
042c89b
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 3, 2019
97393f4
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 4, 2019
5b300c8
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Jan 4, 2019
be995ca
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 7, 2019
b9461c2
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 9, 2019
89f5158
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 10, 2019
2db435e
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 11, 2019
1f3911b
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 14, 2019
3244062
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 14, 2019
89fcbbf
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 14, 2019
71b5c64
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 14, 2019
19cda55
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 14, 2019
c08f7fc
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 15, 2019
dcd95f8
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 16, 2019
f6fa0c2
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 16, 2019
af20179
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 18, 2019
bf32666
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 18, 2019
9386146
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 21, 2019
b7ff595
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 22, 2019
fbbf49d
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 22, 2019
d5e086d
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 23, 2019
9aed507
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 24, 2019
2419bbf
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 24, 2019
1e7e5f4
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 24, 2019
113d1f7
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 24, 2019
b8e918f
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 25, 2019
f11f562
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Jan 28, 2019
bc84e90
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 28, 2019
f3a53c0
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 29, 2019
7a1fb44
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 30, 2019
c49c601
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 30, 2019
a54297f
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 31, 2019
53afa5c
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Jan 31, 2019
18cbfd2
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 1, 2019
5731d47
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 4, 2019
653fee5
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 4, 2019
a4338fc
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 5, 2019
528968e
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 5, 2019
7a33c70
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 5, 2019
4b212dd
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 5, 2019
c4dbfa9
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 6, 2019
48ed802
Merge remote-tracking branch 'upstream/master'
kaiyan-sheng Feb 7, 2019
045c23a
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 8, 2019
70ad00f
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 11, 2019
3d9c02e
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 11, 2019
c14f3a0
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 11, 2019
490e290
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 11, 2019
4c9670f
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 12, 2019
4727c80
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 12, 2019
4be1eaf
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 13, 2019
de9e8c3
Merge branch 'master' of github.com:elastic/beats
kaiyan-sheng Feb 13, 2019
a880cd9
Remove metricNameIDMap and use ListMetrics to construct metric queries
kaiyan-sheng Feb 6, 2019
52da1f5
Rename instanceId to instanceID
kaiyan-sheng Feb 6, 2019
2e847aa
Add utils.go to put common functions for all metricsets
kaiyan-sheng Feb 7, 2019
e148bcf
Add unit test
kaiyan-sheng Feb 8, 2019
ff01451
Remove fields that dont need to apply schema
kaiyan-sheng Feb 8, 2019
4ce70d3
Remove s.Optional in schema and use s.FailOnRequired instead
kaiyan-sheng Feb 8, 2019
6697ac8
Fix unit test
kaiyan-sheng Feb 11, 2019
5e09ea3
Regenerate data,json
kaiyan-sheng Feb 11, 2019
ad112ed
Regenerate data.json
kaiyan-sheng Feb 11, 2019
b3f81cc
Move GetMetricDataResults into utils.go
kaiyan-sheng Feb 13, 2019
0ee5749
Fix unit tests
kaiyan-sheng Feb 13, 2019
5c3707e
Replace with report.error
kaiyan-sheng Feb 13, 2019
89a789f
Separating TestData from TestFetch
kaiyan-sheng Feb 13, 2019
45e0ea8
Move two functions only used by integration tests into a separate file
kaiyan-sheng Feb 14, 2019
e42edb9
Move aws_testing.go under mtest
kaiyan-sheng Feb 19, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions x-pack/metricbeat/module/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
}

// Calculate duration based on period
if config.Period == "" {
err = errors.New("period is not set in AWS module config")
return nil, err
}

durationString, periodSec, err := convertPeriodToDuration(config.Period)
if err != nil {
return nil, err
Expand Down
34 changes: 17 additions & 17 deletions x-pack/metricbeat/module/aws/ec2/_meta/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
"aws": {
"ec2": {
"cpu": {
"credit_balance": 169.089216,
"credit_usage": 0.002926,
"credit_balance": 576,
"credit_usage": 0.144723,
"surplus_credit_balance": 0,
"surplus_credits_charged": 0,
"total": {
"pct": 0.033333333333322
"pct": 1.366194313233248
}
},
"diskio": {
Expand All @@ -27,21 +27,21 @@
},
"instance": {
"core": {
"count": 1
"count": 2
},
"image": {
"id": "ami-01e24be29428c15b2"
"id": "ami-f920cd94"
},
"monitoring": {
"state": "disabled"
},
"private": {
"dns_name": "ip-172-31-26-12.us-west-2.compute.internal",
"ip": "172.31.26.12"
"dns_name": "ip-10-0-0-148.ec2.internal",
"ip": "10.0.0.148"
},
"public": {
"dns_name": "ec2-34-217-213-210.us-west-2.compute.amazonaws.com",
"ip": "34.217.213.210"
"dns_name": "ec2-54-226-109-162.compute-1.amazonaws.com",
"ip": "54.226.109.162"
},
"state": {
"code": 16,
Expand All @@ -51,12 +51,12 @@
},
"network": {
"in": {
"bytes": 296.6,
"packets": 3
"bytes": 737000.4,
"packets": 1361.2
},
"out": {
"bytes": 269,
"packets": 3.6
"bytes": 227871.2,
"packets": 1411.2
}
},
"status": {
Expand All @@ -67,15 +67,15 @@
}
},
"cloud": {
"availability_zone": "us-west-2a",
"availability_zone": "us-east-1b",
"instance": {
"id": "i-077bdaf7e5d81bba3"
"id": "i-77f84332"
},
"machine": {
"type": "t2.micro"
"type": "t2.medium"
},
"provider": "ec2",
"region": "us-west-2"
"region": "us-east-1"
},
"event": {
"dataset": "aws.ec2",
Expand Down
81 changes: 16 additions & 65 deletions x-pack/metricbeat/module/aws/ec2/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
package ec2

import (
"github.com/elastic/beats/libbeat/common"
s "github.com/elastic/beats/libbeat/common/schema"
c "github.com/elastic/beats/libbeat/common/schema/mapstrstr"
)
Expand All @@ -14,85 +13,37 @@ var (
schemaMetricSetFields = s.Schema{
"cpu": s.Object{
"total": s.Object{
"pct": c.Float("cpu.total.pct", s.Optional),
"pct": c.Float("CPUUtilization"),
},
"credit_usage": c.Float("cpu.credit_usage", s.Optional),
"credit_balance": c.Float("cpu.credit_balance", s.Optional),
"surplus_credit_balance": c.Float("cpu.surplus_credit_balance", s.Optional),
"surplus_credits_charged": c.Float("cpu.surplus_credits_charged", s.Optional),
"credit_usage": c.Float("CPUCreditUsage"),
kaiyan-sheng marked this conversation as resolved.
Show resolved Hide resolved
"credit_balance": c.Float("CPUCreditBalance"),
"surplus_credit_balance": c.Float("CPUSurplusCreditBalance"),
"surplus_credits_charged": c.Float("CPUSurplusCreditsCharged"),
},
"diskio": s.Object{
"read": s.Object{
"bytes": c.Float("diskio.read.bytes", s.Optional),
"count": c.Float("diskio.read.count", s.Optional),
"bytes": c.Float("DiskReadBytes"),
"count": c.Float("DiskReadOps"),
},
"write": s.Object{
"bytes": c.Float("diskio.write.bytes", s.Optional),
"count": c.Float("diskio.write.count", s.Optional),
"bytes": c.Float("DiskWriteBytes"),
"count": c.Float("DiskWriteOps"),
},
},
"network": s.Object{
"in": s.Object{
"bytes": c.Float("network.in.bytes", s.Optional),
"packets": c.Float("network.in.packets", s.Optional),
"bytes": c.Float("NetworkIn"),
"packets": c.Float("NetworkPacketsIn"),
},
"out": s.Object{
"bytes": c.Float("network.out.bytes", s.Optional),
"packets": c.Float("network.out.packets", s.Optional),
"bytes": c.Float("NetworkOut"),
"packets": c.Float("NetworkPacketsOut"),
},
},
"status": s.Object{
"check_failed": c.Int("status.check_failed", s.Optional),
"check_failed_instance": c.Int("status.check_failed_instance", s.Optional),
"check_failed_system": c.Int("status.check_failed_system", s.Optional),
},
"instance": s.Object{
"image": s.Object{
"id": c.Str("instance.image.id", s.Optional),
},
"state": s.Object{
"name": c.Str("instance.state.name", s.Optional),
"code": c.Int("instance.state.code", s.Optional),
},
"monitoring": s.Object{
"state": c.Str("instance.monitoring.state", s.Optional),
},
"core": s.Object{
"count": c.Int("instance.core.count", s.Optional),
},
"threads_per_core": c.Int("instance.threads_per_core", s.Optional),
"public": s.Object{
"ip": c.Str("instance.public.ip", s.Optional),
"dns_name": c.Str("instance.public.dns_name", s.Optional),
},
"private": s.Object{
"ip": c.Str("instance.private.ip", s.Optional),
"dns_name": c.Str("instance.private.dns_name", s.Optional),
},
},
}
)

var (
schemaRootFields = s.Schema{
"service": s.Object{
"name": c.Str("service.name", s.Optional),
},
"cloud": s.Object{
"provider": c.Str("cloud.provider", s.Optional),
"availability_zone": c.Str("cloud.availability_zone", s.Optional),
"region": c.Str("cloud.region", s.Optional),
"instance": s.Object{
"id": c.Str("cloud.instance.id", s.Optional),
"name": c.Str("cloud.instance.name", s.Optional),
},
"machine": s.Object{
"type": c.Str("cloud.machine.type", s.Optional),
},
"check_failed": c.Int("StatusCheckFailed"),
"check_failed_instance": c.Int("StatusCheckFailed_Instance"),
"check_failed_system": c.Int("StatusCheckFailed_System"),
},
}
)

func eventMapping(input map[string]interface{}, schema s.Schema) (common.MapStr, error) {
return schema.Apply(input)
}
Loading