diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9786d3152d79..81ec5c9f4cc6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -589,6 +589,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview dashboard for AWS SNS module {pull}14977[14977] - Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100] - Add a `system/service` metricset for systemd data. {pull}14206[14206] +- Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342] - Add cost warnings for the azure module. {pull}15356[15356] - Release elb module as GA. {pull}15485[15485] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fe184abbdd6d..a136a0019476 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2131,84 +2131,84 @@ azure module -*`azure.namespace`*:: -+ --- -The namespace selected +[float] +=== resource +The resource specified -type: keyword --- -*`azure.subscription_id`*:: +*`azure.resource.name`*:: + -- -The subscription ID +The name of the resource type: keyword -- -*`azure.dimensions.*`*:: +*`azure.resource.type`*:: + -- -Azure metric dimensions. +The type of the resource -type: object +type: keyword -- -[float] -=== resource +*`azure.resource.group`*:: ++ +-- +The resource group -The resource specified +type: keyword +-- -*`azure.resource.name`*:: +*`azure.resource.tags.*`*:: + -- -The name of the resource +Azure resource tags. -type: keyword +type: object -- -*`azure.resource.type`*:: +*`azure.namespace`*:: + -- -The type of the resource +The namespace selected type: keyword -- -*`azure.resource.group`*:: +*`azure.subscription_id`*:: + -- -The resource group +The subscription ID type: keyword -- -*`azure.resource.tags.*`*:: +*`azure.dimensions.*`*:: + -- -Azure resource tags. +Azure metric dimensions. type: object -- -*`azure.resource.compute_vm.*.*`*:: +*`azure.compute_vm.*.*`*:: + -- compute_vm @@ -2218,7 +2218,7 @@ type: object -- -*`azure.resource.compute_vm_scaleset.*.*`*:: +*`azure.compute_vm_scaleset.*.*`*:: + -- compute_vm_scaleset @@ -2235,12 +2235,22 @@ monitor -*`azure.resource.monitor.metrics.*.*`*:: +*`azure.monitor.metrics.*.*`*:: + -- Metrics returned. +type: object + +-- + +*`azure.storage.*.*`*:: ++ +-- +storage account + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index b37998ad191d..8d69c8e5bd1b 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -78,6 +78,11 @@ so the `period` for `compute_vm` metricset should be `300s` or multiples of `30 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes, so the `period` for `compute_vm_scaleset` metricset should be `300s` or multiples of `300s`. +[float] +=== `storage` +This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes, +so the `period` for `storage` metricset should be `300s` or multiples of `300s`. + [float] == Additional notes about metrics and costs @@ -130,6 +135,16 @@ metricbeat.modules: client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - storage + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' ---- [float] @@ -143,9 +158,13 @@ The following metricsets are available: * <> +* <> + include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] include::azure/monitor.asciidoc[] +include::azure/storage.asciidoc[] + diff --git a/metricbeat/docs/modules/azure/storage.asciidoc b/metricbeat/docs/modules/azure/storage.asciidoc new file mode 100644 index 000000000000..21a073d0b067 --- /dev/null +++ b/metricbeat/docs/modules/azure/storage.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-storage]] +=== azure storage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 6a627a663537..b536d117c5b3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -28,9 +28,10 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> beta[] +.4+| .4+| |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 21b79e9777ad..0625c8daf96d 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -22,6 +22,7 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm_scaleset" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/monitor" + _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/cockroachdb" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns" _ "github.com/elastic/beats/x-pack/metricbeat/module/coredns/stats" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1669ac259859..d9f5c41f435a 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -281,6 +281,16 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - storage + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 92051e4271cc..8b3a6e64ec6e 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -32,3 +32,13 @@ client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - storage + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 14ad9fe79c44..009159d21b34 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -35,3 +35,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - storage +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index b683caf17d96..469ed19c6cb7 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -68,6 +68,11 @@ so the `period` for `compute_vm` metricset should be `300s` or multiples of `30 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes, so the `period` for `compute_vm_scaleset` metricset should be `300s` or multiples of `300s`. +[float] +=== `storage` +This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes, +so the `period` for `storage` metricset should be `300s` or multiples of `300s`. + [float] == Additional notes about metrics and costs diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index 535fd4d431fc..427f33b9c995 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -4,44 +4,44 @@ azure module release: beta fields: - - name: azure - type: group - description: > - fields: - - name: namespace - type: keyword - description: > - The namespace selected - - name: subscription_id - type: keyword - description: > - The subscription ID - - name: dimensions.* - type: object - object_type: keyword - object_type_mapping_type: "*" - description: > - Azure metric dimensions. - - name: resource - type: group - description: > - The resource specified - fields: - - name: name - type: keyword + - name: azure + type: group + description: > + fields: + - name: resource + type: group description: > - The name of the resource - - name: type + The resource specified + fields: + - name: name + type: keyword + description: > + The name of the resource + - name: type + type: keyword + description: > + The type of the resource + - name: group + type: keyword + description: > + The resource group + - name: tags.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Azure resource tags. + - name: namespace type: keyword description: > - The type of the resource - - name: group + The namespace selected + - name: subscription_id type: keyword description: > - The resource group - - name: tags.* + The subscription ID + - name: dimensions.* type: object object_type: keyword object_type_mapping_type: "*" description: > - Azure resource tags. + Azure metric dimensions. diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json index c7a185840fd1..873616c609ca 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json @@ -361,7 +361,7 @@ "line_width": "1", "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_running.avg", + "field": "azure.compute_vm.asp_net_applications_running.avg", "id": "d1acb8f2-eaa2-11e9-a229-c9171499dcc6", "type": "max" } @@ -436,7 +436,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.sqlserver.general_statistics_user_connections.avg", + "field": "azure.compute_vm.sqlserver_general_statistics_user_connections.avg", "id": "da495db2-eaa7-11e9-a88b-4b683ca3087b", "type": "avg" } @@ -510,7 +510,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_requests_timed_out.avg", + "field": "azure.compute_vm.asp_net_applications_requests_timed_out.avg", "id": "be74e9e2-eaa4-11e9-8923-850d87d8e766", "type": "avg" } @@ -532,7 +532,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_requests_failed.avg", + "field": "azure.compute_vm.asp_net_applications_requests_failed.avg", "id": "be74e9e4-eaa4-11e9-8923-850d87d8e766", "type": "avg" } @@ -554,7 +554,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_requests_succeeded.avg", + "field": "azure.compute_vm.asp_net_applications_requests_succeeded.avg", "id": "be7510f1-eaa4-11e9-8923-850d87d8e766", "type": "avg" } @@ -576,7 +576,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_requests_total.avg", + "field": "azure.compute_vm.asp_net_applications_requests_total.avg", "id": "be7510f3-eaa4-11e9-8923-850d87d8e766", "type": "avg" } @@ -654,7 +654,7 @@ "line_width": "1", "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_errors_total.avg", + "field": "azure.compute_vm.asp_net_applications_errors_total.avg", "id": "29578b11-eaa4-11e9-a2d3-e7a00bbd3c18", "type": "avg" } @@ -728,7 +728,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.sqlserver.memory_manager_total_server_memory.avg", + "field": "azure.compute_vm.sqlserver_memory_manager_total_server_memory.avg", "id": "94af6a02-eaa8-11e9-9269-d92e2d3f77fd", "type": "avg" } @@ -802,7 +802,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_sessions_active.avg", + "field": "azure.compute_vm.asp_net_applications_sessions_active.avg", "id": "6d6575a2-eaa5-11e9-84ad-5919a47b8f34", "type": "avg" } @@ -824,7 +824,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_sessions_timed_out.avg", + "field": "azure.compute_vm.asp_net_applications_sessions_timed_out.avg", "id": "6d6575a4-eaa5-11e9-84ad-5919a47b8f34", "type": "avg" } @@ -846,7 +846,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_sessions_abandoned.avg", + "field": "azure.compute_vm.asp_net_applications_sessions_abandoned.avg", "id": "6d6575a6-eaa5-11e9-84ad-5919a47b8f34", "type": "avg" } @@ -868,7 +868,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.asp.net_applications_sessions_total.avg", + "field": "azure.compute_vm.asp_net_applications_sessions_total.avg", "id": "6d6575a8-eaa5-11e9-84ad-5919a47b8f34", "type": "avg" } @@ -942,7 +942,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.sqlserver.buffer_manager_page_reads_per_sec.avg", + "field": "azure.compute_vm.sqlserver_buffer_manager_page_reads_per_sec.avg", "id": "35459a32-eaa8-11e9-a379-c33a712c0373", "type": "avg" } @@ -964,7 +964,7 @@ "line_width": 2, "metrics": [ { - "field": "azure.compute_vm.sqlserver.buffer_manager_page_writes_per_sec.avg", + "field": "azure.compute_vm.sqlserver_buffer_manager_page_writes_per_sec.avg", "id": "35459a34-eaa8-11e9-a379-c33a712c0373", "type": "avg" } diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index c98dd4448277..2da534d7372d 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -27,11 +27,12 @@ type Config struct { // ResourceConfig contains resource and metric list specific configuration. type ResourceConfig struct { - ID []string `config:"resource_id"` - Group []string `config:"resource_group"` - Metrics []MetricConfig `config:"metrics"` - Type string `config:"resource_type"` - Query string `config:"resource_query"` + ID []string `config:"resource_id"` + Group []string `config:"resource_group"` + Metrics []MetricConfig `config:"metrics"` + Type string `config:"resource_type"` + Query string `config:"resource_query"` + ServiceType []string `config:"service_type"` } // MetricConfig contains metric specific configuration. @@ -72,8 +73,8 @@ func newModule(base mb.BaseModule) (mb.Module, error) { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - Client *Client - MapMetric mapMetric + Client *Client + MapMetrics mapResourceMetrics } // NewMetricSet will instantiate a new azure metricset @@ -117,7 +118,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { - err := m.Client.InitResources(m.MapMetric, report) + err := m.Client.InitResources(m.MapMetrics, report) if err != nil { return err } @@ -126,9 +127,14 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } // retrieve metrics - err = m.Client.GetMetricValues(report) - if err != nil { - return err + groupedMetrics := groupMetricsByResource(m.Client.Resources.Metrics) + + for _, metrics := range groupedMetrics { + results := m.Client.GetMetricValues(metrics, report) + err := EventsMapping(results, m.BaseMetricSet.Name(), report) + if err != nil { + return errors.Wrap(err, "error running EventsMapping") + } } - return EventsMapping(report, m.Client.Resources.Metrics, m.BaseMetricSet.Name()) + return nil } diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 36723b19b9b1..b200daa3c772 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -26,8 +26,8 @@ type Client struct { Log *logp.Logger } -// mapMetric function type will map the configuration options to client metrics (depending on the metricset) -type mapMetric func(client *Client, metric MetricConfig, resource resources.GenericResource) ([]Metric, error) +// mapResourceMetrics function type will map the configuration options to client metrics (depending on the metricset) +type mapResourceMetrics func(client *Client, resources []resources.GenericResource, resourceConfig ResourceConfig) ([]Metric, error) // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { @@ -46,7 +46,7 @@ func NewClient(config Config) (*Client, error) { // InitResources function will retrieve and validate the resources configured by the users and then map the information configured to client metrics. // the mapMetric function sent in this case will handle the mapping part as different metric and aggregation options work for different metricsets -func (client *Client) InitResources(fn mapMetric, report mb.ReporterV2) error { +func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) error { if len(client.Config.Resources) == 0 { return errors.New("no resource options defined") } @@ -68,15 +68,11 @@ func (client *Client) InitResources(fn mapMetric, report mb.ReporterV2) error { client.Log.Error(err) continue } - for _, res := range resourceList.Values() { - for _, metric := range resource.Metrics { - met, err := fn(client, metric, res) - if err != nil { - return err - } - metrics = append(metrics, met...) - } + resourceMetrics, err := fn(client, resourceList.Values(), resource) + if err != nil { + return err } + metrics = append(metrics, resourceMetrics...) } // users could add or remove resources while metricbeat is running so we could encounter the situation where resources are unavailable we log an error message (see above) // we also log a debug message when absolutely no resources are found @@ -88,12 +84,18 @@ func (client *Client) InitResources(fn mapMetric, report mb.ReporterV2) error { } // GetMetricValues returns the specified metric data points for the specified resource ID/namespace. -func (client *Client) GetMetricValues(report mb.ReporterV2) error { +func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) []Metric { + var resultedMetrics []Metric // loop over the set of metrics - for i, metric := range client.Resources.Metrics { + for _, metric := range metrics { // select period to collect metrics, will double the interval value in order to retrieve any missing values + //if timegrain is larger than intervalx2 then interval will be assigned the timegrain value + interval := client.Config.Period + if t := convertTimegrainToDuration(metric.TimeGrain); t > interval*2 { + interval = t + } endTime := time.Now().UTC() - startTime := endTime.Add(client.Config.Period * (-2)) + startTime := endTime.Add(interval * (-2)) timespan := fmt.Sprintf("%s/%s", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) // build the 'filter' parameter which will contain any dimensions configured @@ -105,29 +107,30 @@ func (client *Client) GetMetricValues(report mb.ReporterV2) error { } filter = strings.Join(filterList, " AND ") } - resp, err := client.AzureMonitorService.GetMetricValues(metric.Resource.ID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, + resp, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { - err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.ID, metric.Namespace) - client.LogError(report, err) + err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubID, metric.Namespace) + client.Log.Error(err) + report.Error(err) } else { - current := mapMetricValues(resp, client.Resources.Metrics[i].Values, endTime.Truncate(time.Minute).Add(client.Config.Period*(-1)), endTime.Truncate(time.Minute)) - client.Resources.Metrics[i].Values = current + for i, currentMetric := range client.Resources.Metrics { + if matchMetrics(currentMetric, metric) { + current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) + client.Resources.Metrics[i].Values = current + resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) + } + } } } - return nil -} - -// LogError is used to reduce the number of lines written when logging errors -func (client *Client) LogError(report mb.ReporterV2, err error) { - client.Log.Error(err) - report.Error(err) + return resultedMetrics } // CreateMetric function will create a client metric based on the resource and metrics configured -func (client *Client) CreateMetric(resource resources.GenericResource, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { +func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { met := Metric{ Resource: Resource{ + SubID: selectedResourceID, ID: *resource.ID, Name: *resource.Name, Location: *resource.Location, @@ -150,7 +153,7 @@ func (client *Client) CreateMetric(resource resources.GenericResource, namespace } // MapMetricByPrimaryAggregation will map the primary aggregation of the metric definition to the client metric -func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefinition, resource resources.GenericResource, namespace string, dim []Dimension, timegrain string) []Metric { +func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, namespace string, dim []Dimension, timegrain string) []Metric { var clientMetrics []Metric metricGroups := make(map[string][]insights.MetricDefinition) @@ -162,7 +165,10 @@ func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefi for _, metricName := range metricGroup { metricNames = append(metricNames, *metricName.Name.Value) } - clientMetrics = append(clientMetrics, client.CreateMetric(resource, namespace, metricNames, key, dim, timegrain)) + if selectedResourceID == "" { + selectedResourceID = *resource.ID + } + clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, namespace, metricNames, key, dim, timegrain)) } return clientMetrics } diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index a4d9a9512948..f79f9728df88 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -36,7 +36,7 @@ var ( } ) -func mockMapMetric(client *Client, metric MetricConfig, resource resources.GenericResource) ([]Metric, error) { +func mockMapResourceMetrics(client *Client, resources []resources.GenericResource, resourceConfig ResourceConfig) ([]Metric, error) { return nil, nil } @@ -44,7 +44,7 @@ func TestInitResources(t *testing.T) { t.Run("return error when no resource options were configured", func(t *testing.T) { client := NewMockClient() mr := MockReporterV2{} - err := client.InitResources(mockMapMetric, &mr) + err := client.InitResources(mockMapResourceMetrics, &mr) assert.Error(t, err, "no resource options were configured") }) t.Run("return error no resources were found", func(t *testing.T) { @@ -55,7 +55,7 @@ func TestInitResources(t *testing.T) { client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - err := client.InitResources(mockMapMetric, &mr) + err := client.InitResources(mockMapResourceMetrics, &mr) assert.Error(t, err, "no resources were found based on all the configurations options entered") assert.Equal(t, len(client.Resources.Metrics), 0) m.AssertExpectations(t) @@ -65,6 +65,7 @@ func TestInitResources(t *testing.T) { func TestGetMetricValues(t *testing.T) { client := NewMockClient() client.Config = resourceIDConfig + t.Run("return no error when no metric values are returned but log and send event", func(t *testing.T) { client.Resources = ResourceConfiguration{ Metrics: []Metric{ @@ -77,13 +78,13 @@ func TestGetMetricValues(t *testing.T) { }, } m := &MockService{} - m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once(). Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found")) client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - err := client.GetMetricValues(&mr) - assert.Nil(t, err) + metrics := client.GetMetricValues(client.Resources.Metrics, &mr) + assert.Equal(t, len(metrics), 0) assert.Equal(t, len(client.Resources.Metrics[0].Values), 0) m.AssertExpectations(t) }) @@ -99,13 +100,13 @@ func TestGetMetricValues(t *testing.T) { }, } m := &MockService{} - m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found")) client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - err := client.GetMetricValues(&mr) - assert.Nil(t, err) + metricValues := client.GetMetricValues(client.Resources.Metrics, &mr) + assert.Equal(t, len(metricValues), 0) assert.Equal(t, len(client.Resources.Metrics[0].Values), 0) m.AssertExpectations(t) }) diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index 290f0168b5ba..b03964044ea2 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -151,3 +151,40 @@ func compareMetricValues(metVal *float64, metricVal *float64) bool { } return false } + +// convertTimegrainToDuration will convert azure timegrain options to actual duration values +func convertTimegrainToDuration(timegrain string) time.Duration { + var duration time.Duration + switch timegrain { + case "PT1M": + duration = time.Duration(time.Minute) + default: + case "PT5M": + duration = time.Duration(5 * time.Minute) + case "PT15M": + duration = time.Duration(15 * time.Minute) + case "PT30M": + duration = time.Duration(30 * time.Minute) + case "PT1H": + duration = time.Duration(time.Hour) + case "PT6H": + duration = time.Duration(6 * time.Hour) + case "PT12H": + duration = time.Duration(12 * time.Hour) + case "PT1D": + duration = time.Duration(24 * time.Hour) + } + return duration +} + +// groupMetricsByResource is used in order to group metrics by resource and return data faster +func groupMetricsByResource(metrics []Metric) map[string][]Metric { + grouped := make(map[string][]Metric) + for _, metric := range metrics { + if _, ok := grouped[metric.Resource.ID]; !ok { + grouped[metric.Resource.ID] = make([]Metric, 0) + } + grouped[metric.Resource.ID] = append(grouped[metric.Resource.ID], metric) + } + return grouped +} diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go index 9c48381b6a8f..d4b4e241af45 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go @@ -12,33 +12,34 @@ import ( "github.com/elastic/beats/x-pack/metricbeat/module/azure" ) -// mapMetric should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetric(client *azure.Client, metric azure.MetricConfig, resource resources.GenericResource) ([]azure.Metric, error) { +// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters +func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric - //check if no metric names are configured - if metric.Name == nil { + if len(resourceConfig.Metrics) == 0 { return nil, nil } - // return all namespaces supported for this resource - namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) - if err != nil { - return nil, errors.Wrapf(err, "no metric namespaces were found for resource %s", *resource.ID) - } - for _, namespace := range *namespaces.Value { - // get all metric definitions supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, *namespace.Properties.MetricNamespaceName) + for _, resource := range resources { + // return all namespaces supported for this resource + namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) - } - if len(*metricDefinitions.Value) == 0 { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) + return nil, errors.Wrapf(err, "no metric namespaces were found for resource %s", *resource.ID) } - var filteredMetricDefinitions []insights.MetricDefinition - for _, metricDefinition := range *metricDefinitions.Value { - filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) + for _, namespace := range *namespaces.Value { + // get all metric definitions supported by the namespace provided + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, *namespace.Properties.MetricNamespaceName) + if err != nil { + return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) + } + if len(*metricDefinitions.Value) == 0 { + return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) + } + var filteredMetricDefinitions []insights.MetricDefinition + for _, metricDefinition := range *metricDefinitions.Value { + filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) + } + // map azure metric definitions to client metrics + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, filteredMetricDefinitions, resource, "", *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) } - // map azure metric definitions to client metrics - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, filteredMetricDefinitions, resource, *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) } return metrics, nil } diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go index 624668505054..1d24f6b65aaf 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go @@ -79,12 +79,13 @@ func TestMapMetric(t *testing.T) { Value: &emptyList, } metricConfig := azure.MetricConfig{Name: []string{"*"}} + var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} client := azure.NewMockClient() t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { m := &azure.MockService{} m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) client.AzureMonitorService = m - metric, err := mapMetric(client, metricConfig, resource) + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NotNil(t, err) assert.Equal(t, err.Error(), "no metric namespaces were found for resource 123: invalid resource ID") assert.Equal(t, metric, []azure.Metric(nil)) @@ -95,7 +96,7 @@ func TestMapMetric(t *testing.T) { m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) client.AzureMonitorService = m - metric, err := mapMetric(client, metricConfig, resource) + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NotNil(t, err) assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") assert.Equal(t, metric, []azure.Metric(nil)) @@ -106,7 +107,7 @@ func TestMapMetric(t *testing.T) { m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m - metrics, err := mapMetric(client, metricConfig, resource) + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) assert.Equal(t, metrics[0].Resource.ID, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go index 526562c3f2dc..2b9774c2a29e 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go @@ -56,7 +56,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, } } - ms.MapMetric = mapMetric + ms.MapMetrics = mapMetrics return &MetricSet{ MetricSet: ms, }, nil diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index 5eacb816078b..aa5e699109f9 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -19,64 +19,67 @@ const ( customVMDimension = "VirtualMachine" defaultSlotIDDimension = "SlotId" defaultTimeGrain = "PT5M" - noDimension = "none" ) -// mapMetric should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetric(client *azure.Client, metric azure.MetricConfig, resource resources.GenericResource) ([]azure.Metric, error) { +// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters +func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) - if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s", *resource.ID, metric.Namespace) - } - if len(*metricDefinitions.Value) == 0 && metric.Namespace != customVMNamespace { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) - } - var supportedMetricNames []insights.MetricDefinition - if strings.Contains(strings.Join(metric.Name, " "), "*") { - for _, definition := range *metricDefinitions.Value { - supportedMetricNames = append(supportedMetricNames, definition) - } - } else { - // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names - for _, name := range metric.Name { - for _, metricDefinition := range *metricDefinitions.Value { - if name == *metricDefinition.Name.Value { - supportedMetricNames = append(supportedMetricNames, metricDefinition) + for _, resource := range resources { + for _, metric := range resourceConfig.Metrics { + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) + if err != nil { + return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s", *resource.ID, metric.Namespace) + } + if len(*metricDefinitions.Value) == 0 && metric.Namespace != customVMNamespace { + return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) + } + var supportedMetricNames []insights.MetricDefinition + if strings.Contains(strings.Join(metric.Name, " "), "*") { + for _, definition := range *metricDefinitions.Value { + supportedMetricNames = append(supportedMetricNames, definition) + } + } else { + // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names + for _, name := range metric.Name { + for _, metricDefinition := range *metricDefinitions.Value { + if name == *metricDefinition.Name.Value { + supportedMetricNames = append(supportedMetricNames, metricDefinition) + } + } } } + if len(supportedMetricNames) == 0 { + continue + } + groupedMetrics := make(map[string][]insights.MetricDefinition) + var vmdim string + if metric.Namespace == defaultVMScalesetNamespace { + vmdim = defaultVMDimension + } else if metric.Namespace == customVMNamespace { + vmdim = customVMDimension + } + for _, metricName := range supportedMetricNames { + if metricName.Dimensions == nil || len(*metricName.Dimensions) == 0 { + groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) + } else if containsDimension(vmdim, *metricName.Dimensions) { + groupedMetrics[vmdim] = append(groupedMetrics[vmdim], metricName) + } else if containsDimension(defaultSlotIDDimension, *metricName.Dimensions) { + groupedMetrics[defaultSlotIDDimension] = append(groupedMetrics[defaultSlotIDDimension], metricName) + } + } + for key, metricGroup := range groupedMetrics { + var metricNameList []string + for _, metricName := range metricGroup { + metricNameList = append(metricNameList, *metricName.Name.Value) + } + var dimensions []azure.Dimension + if key != azure.NoDimension { + dimensions = []azure.Dimension{{Name: key, Value: "*"}} + } + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, defaultTimeGrain)...) + } } } - if len(supportedMetricNames) == 0 { - return nil, nil - } - groupedMetrics := make(map[string][]insights.MetricDefinition) - var vmdim string - if metric.Namespace == defaultVMScalesetNamespace { - vmdim = defaultVMDimension - } else if metric.Namespace == customVMNamespace { - vmdim = customVMDimension - } - for _, metricName := range supportedMetricNames { - if metricName.Dimensions == nil || len(*metricName.Dimensions) == 0 { - groupedMetrics[noDimension] = append(groupedMetrics[noDimension], metricName) - } else if containsDimension(vmdim, *metricName.Dimensions) { - groupedMetrics[vmdim] = append(groupedMetrics[vmdim], metricName) - } else if containsDimension(defaultSlotIDDimension, *metricName.Dimensions) { - groupedMetrics[defaultSlotIDDimension] = append(groupedMetrics[defaultSlotIDDimension], metricName) - } - } - for key, metricGroup := range groupedMetrics { - var metricNameList []string - for _, metricName := range metricGroup { - metricNameList = append(metricNameList, *metricName.Name.Value) - } - var dimensions []azure.Dimension - if key != noDimension { - dimensions = []azure.Dimension{{Name: key, Value: "*"}} - } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, metric.Namespace, dimensions, defaultTimeGrain)...) - } return metrics, nil } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index 05cd898c2d48..889c029a0e9d 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -63,12 +63,13 @@ func TestMapMetric(t *testing.T) { Value: &emptyList, } metricConfig := azure.MetricConfig{Name: []string{"*"}, Namespace: "namespace"} + var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} client := azure.NewMockClient() t.Run("return error when the metric metric definition api call returns an error", func(t *testing.T) { m := &azure.MockService{} m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, errors.New("invalid resource ID")) client.AzureMonitorService = m - metric, err := mapMetric(client, metricConfig, resource) + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NotNil(t, err) assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace: invalid resource ID") assert.Equal(t, metric, []azure.Metric(nil)) @@ -78,7 +79,7 @@ func TestMapMetric(t *testing.T) { m := &azure.MockService{} m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) client.AzureMonitorService = m - metric, err := mapMetric(client, metricConfig, resource) + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NotNil(t, err) assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") assert.Equal(t, metric, []azure.Metric(nil)) @@ -88,7 +89,7 @@ func TestMapMetric(t *testing.T) { m := &azure.MockService{} m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m - metrics, err := mapMetric(client, metricConfig, resource) + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) assert.Equal(t, len(metrics), 2) diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go index e2ae41b69f72..7b321e3c0797 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go @@ -66,7 +66,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, } } - ms.MapMetric = mapMetric + ms.MapMetrics = mapMetrics return &MetricSet{ MetricSet: ms, }, nil diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index ed690593d6b4..5b5ebc04eed9 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -14,12 +14,13 @@ import ( ) const ( - noDimension = "none" + // NoDimension is used to group metrics in separate api calls in order to reduce the number of executions + NoDimension = "none" nativeMetricset = "monitor" ) // EventsMapping will map metric values to beats events -func EventsMapping(report mb.ReporterV2, metrics []Metric, metricset string) error { +func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) error { // metrics and metric values are currently grouped relevant to the azure REST API calls (metrics with the same aggregations per call) // multiple metrics can be mapped in one event depending on the resource, namespace, dimensions and timestamp @@ -39,7 +40,7 @@ func EventsMapping(report mb.ReporterV2, metrics []Metric, metricset string) err for resNamKey, resourceMetrics := range groupByResourceNamespace { for _, resourceMetric := range resourceMetrics { if len(resourceMetric.Dimensions) == 0 { - groupByDimensions[resNamKey+noDimension] = append(groupByDimensions[resNamKey+noDimension], resourceMetric) + groupByDimensions[resNamKey+NoDimension] = append(groupByDimensions[resNamKey+NoDimension], resourceMetric) } else { var dimKey string for _, dim := range resourceMetric.Dimensions { @@ -61,6 +62,8 @@ func EventsMapping(report mb.ReporterV2, metrics []Metric, metricset string) err } } for timestamp, groupTimeValues := range groupByTimeMetrics { + var event mb.Event + var metricList common.MapStr // group events by dimension values exists, validDimensions := returnAllDimensions(defaultMetric.Dimensions) if exists { @@ -71,16 +74,22 @@ func EventsMapping(report mb.ReporterV2, metrics []Metric, metricset string) err groupByDimensions[dimKey] = append(groupByDimensions[dimKey], dimGroupValue) } for _, groupDimValues := range groupByDimensions { - report.Event(initEvent(timestamp, defaultMetric, groupDimValues, metricset)) + event, metricList = createEvent(timestamp, defaultMetric, groupDimValues) } } } else { - report.Event(initEvent(timestamp, defaultMetric, groupTimeValues, metricset)) + event, metricList = createEvent(timestamp, defaultMetric, groupTimeValues) } + if metricset == nativeMetricset { + event.ModuleFields.Put("metrics", metricList) + } else { + for key, metric := range metricList { + event.MetricSetFields.Put(key, metric) + } + } + report.Event(event) } - } - return nil } @@ -94,7 +103,7 @@ func managePropertyName(metric string) string { // replace actual percentage symbol with the smbol "pct" resultMetricName = strings.Replace(resultMetricName, "_%_", "_pct_", -1) // create an object in case of ":" - resultMetricName = strings.Replace(resultMetricName, ":", ".", -1) + resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) // create an object in case of ":" resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" @@ -104,32 +113,13 @@ func managePropertyName(metric string) string { obj[index] = strings.TrimPrefix(obj[index], "_") obj[index] = strings.TrimSuffix(obj[index], "_") } - resultMetricName = strings.ToLower(strings.Join(obj, ".")) + resultMetricName = strings.ToLower(strings.Join(obj, "_")) return resultMetricName } -// initEvent will create a new base event -func initEvent(timestamp time.Time, metric Metric, metricValues []MetricValue, metricset string) mb.Event { - metricList := common.MapStr{} - for _, value := range metricValues { - metricNameString := fmt.Sprintf("%s", managePropertyName(value.name)) - if value.min != nil { - metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "min"), *value.min) - } - if value.max != nil { - metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "max"), *value.max) - } - if value.avg != nil { - metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "avg"), *value.avg) - } - if value.total != nil { - metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "total"), *value.total) - } - if value.count != nil { - metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "count"), *value.count) - } - } +// createEvent will create a new base event +func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) { event := mb.Event{ ModuleFields: common.MapStr{ "resource": common.MapStr{ @@ -156,18 +146,30 @@ func initEvent(timestamp time.Time, metric Metric, metricValues []MetricValue, m } } - if metricset == nativeMetricset { - event.ModuleFields.Put("metrics", metricList) - } else { - for key, metric := range metricList { - event.MetricSetFields.Put(key, metric) - } - } - event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") event.RootFields.Put("cloud.region", metric.Resource.Location) - return event + + metricList := common.MapStr{} + for _, value := range metricValues { + metricNameString := fmt.Sprintf("%s", managePropertyName(value.name)) + if value.min != nil { + metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "min"), *value.min) + } + if value.max != nil { + metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "max"), *value.max) + } + if value.avg != nil { + metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "avg"), *value.avg) + } + if value.total != nil { + metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "total"), *value.total) + } + if value.count != nil { + metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "count"), *value.count) + } + } + return event, metricList } // getDimensionValue will return dimension value for the key provided diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index b3b34b76ef66..8c678616d180 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVL1ugzAQ3nmKE2Ok5AEYKlXq0qFbd+TYH6kbwJZtWqVPXwGJDYZAk0z1gGSb+/7u5C0dccqI/TQGCZGTrkRGabdPEyIBy43UTqo6o6eEiPp/qVKiKdsSgxLMIqM9HEuIColS2Cwh2lLNKgRwInfSyOhgVKO7/Qx6KG/XBaL9Ws04zucXqCNO38oIfzoD2K/3DwQQsijBHUTEYpu9L8+leIRrCEWvLxGRkBVqK1Vtd5uIRe0/wZ0/7Lf5vILBZV4xrWV9OP+ZbtJ1nc99I+GM5ENJkVgDqxoziT50cTWMCwJZDS4LiWBh3O5xwwco8/4XaEPHSRXkBiImVC3241Rt2SrVOLO7uXyeYzxviB2Gc3VlslZm66/TtSi2nzAvtxM2kctVpRuH/KvabUay45flFjtFqdiVy3vNBJ0LFnLLWQkL91+8eMH+fuKuUrV0ykxUx9N8zeOCjil0/CCMhHRPlY2iXYhxNcpb4lyx0q63XiAZuMbUEGHYk98AAAD//xK8B3k=" + return "eJzUVDtPwzAQ3vMrThkrtT8gAxISCwMbe+Tal2AaP+QHqPx65KR1nXcLHcCDpfjO3+Pu4i0c8FgA+fIGMwDHXYMF5O13ngEwtNRw7biSBTxkANDlglDMN+GKwQaJxQL26EgGUHFsmC3a1C1IIvACH5Y7aiygNsrr08kERx8mhTJolTcUY2AKcRa1W69vGHHAaqS84siSlCF5KiDsvcBZwAGPn8qwQWxBxllKQARVgUtkTVIHnvtSh+tXUQ+rewfu2IExdjRMarvbTNKq/TtSNwh1h+WSsCSlFERrLutTfr7JbzPx2P4I0UYrdjSvYbeaTAzsWN/KyEYosNggdcnIntms30eIkrPfc6aA8Pw0ImRcoLRcyX6fZnq00p9re7OgueuJQGc4TcWNhFMltHdYfojdpid9+J7dYqlqFJkJ/tTQReeChdJS0qBF91+8RMExPnInlOROmdV3fs7jgo4x9NKD3w2THZR2oYyrpbylnCtWwnrpBIJB541EtpsvqnXKkBr/9picRAKhVHnpsu8AAAD//wCRS6g=" } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 0ac917b8c2b6..2a6ab1495e30 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -7,84 +7,81 @@ package monitor import ( "strings" + "github.com/pkg/errors" + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - "github.com/elastic/beats/x-pack/metricbeat/module/azure" ) -// mapMetric should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetric(client *azure.Client, metric azure.MetricConfig, resource resources.GenericResource) ([]azure.Metric, error) { +// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters +func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric - // get all metrics supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) - if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) - } - if len(*metricDefinitions.Value) == 0 { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) + for _, resource := range resources { + for _, metric := range resourceConfig.Metrics { + // get all metrics supported by the namespace provided + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) + if err != nil { + return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) + } + if len(*metricDefinitions.Value) == 0 { + return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) + } + + // validate metric names and filter on the supported metrics + supportedMetricNames, err := filterMetricNames(*resource.ID, metric, *metricDefinitions.Value) + if err != nil { + return nil, err + } + + //validate aggregations and filter on supported aggregations + metricGroups, err := filterOnSupportedAggregations(supportedMetricNames, metric, *metricDefinitions.Value) + if err != nil { + return nil, err + } + + // map dimensions + var dim []azure.Dimension + if len(metric.Dimensions) > 0 { + for _, dimension := range metric.Dimensions { + dim = append(dim, azure.Dimension{Name: dimension.Name, Value: dimension.Value}) + } + } + for key, metricGroup := range metricGroups { + var metricNames []string + for _, metricName := range metricGroup { + metricNames = append(metricNames, *metricName.Name.Value) + } + metrics = append(metrics, client.CreateMetric(*resource.ID, resource, metric.Namespace, metricNames, key, dim, metric.Timegrain)) + } + } } + return metrics, nil +} - // validate metric names - // check if all metric names are selected (*) +// filterMetricNames func will verify if the metric names entered are valid and will also return the corresponding list of metrics +func filterMetricNames(resourceID string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) ([]string, error) { var supportedMetricNames []string var unsupportedMetricNames []string - if strings.Contains(strings.Join(metric.Name, " "), "*") { - for _, definition := range *metricDefinitions.Value { + // check if all metric names are selected (*) + if strings.Contains(strings.Join(metricConfig.Name, " "), "*") { + for _, definition := range metricDefinitions { supportedMetricNames = append(supportedMetricNames, *definition.Name.Value) } } else { // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names - supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metric.Name, *metricDefinitions.Value) + supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions) if len(unsupportedMetricNames) > 0 { return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s", - strings.Join(unsupportedMetricNames, ","), *resource.ID, metric.Namespace) + strings.Join(unsupportedMetricNames, ","), resourceID, metricConfig.Namespace) } } if len(supportedMetricNames) == 0 { - return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metric.Name, ","), *resource.ID, metric.Namespace) + return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceID, metricConfig.Namespace) } - //validate aggregations and filter on supported ones - var supportedAggregations []string - var unsupportedAggregations []string - metricGroups := make(map[string][]insights.MetricDefinition) - metricDefs := getMetricDefinitionsByNames(*metricDefinitions.Value, supportedMetricNames) - - if len(metric.Aggregations) == 0 { - for _, metricDef := range metricDefs { - metricGroups[string(metricDef.PrimaryAggregationType)] = append(metricGroups[string(metricDef.PrimaryAggregationType)], metricDef) - } - } else { - supportedAggregations, unsupportedAggregations = filterAggregations(metric.Aggregations, metricDefs) - if len(unsupportedAggregations) > 0 { - return nil, errors.Errorf("the aggregations configured : %s are not supported for some of the metrics selected %s ", - strings.Join(unsupportedAggregations, ","), strings.Join(supportedMetricNames, ",")) - } - if len(supportedAggregations) == 0 { - return nil, errors.Errorf("no aggregations were found based on the aggregation values configured or supported between the metrics : %s", - strings.Join(supportedMetricNames, ",")) - } - key := strings.Join(supportedAggregations, ",") - metricGroups[key] = append(metricGroups[key], metricDefs...) - } - // map dimensions - var dim []azure.Dimension - if len(metric.Dimensions) > 0 { - for _, dimension := range metric.Dimensions { - dim = append(dim, azure.Dimension{Name: dimension.Name, Value: dimension.Value}) - } - } - for key, metricGroup := range metricGroups { - var metricNames []string - for _, metricName := range metricGroup { - metricNames = append(metricNames, *metricName.Name.Value) - } - metrics = append(metrics, client.CreateMetric(resource, metric.Namespace, metricNames, key, dim, metric.Timegrain)) - } - return metrics, nil + return supportedMetricNames, nil } // filterSConfiguredMetrics will filter out any unsupported metrics based on the namespace selected @@ -101,11 +98,37 @@ func filterSConfiguredMetrics(selectedRange []string, allRange []insights.Metric } else { notInRange = append(notInRange, name) } - } return inRange, notInRange } +// filterOnSupportedAggregations will verify if the aggregation values entered are supported and will also return the corresponding list of aggregations +func filterOnSupportedAggregations(metricNames []string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) (map[string][]insights.MetricDefinition, error) { + var supportedAggregations []string + var unsupportedAggregations []string + metricGroups := make(map[string][]insights.MetricDefinition) + metricDefs := getMetricDefinitionsByNames(metricDefinitions, metricNames) + + if len(metricConfig.Aggregations) == 0 { + for _, metricDef := range metricDefs { + metricGroups[string(metricDef.PrimaryAggregationType)] = append(metricGroups[string(metricDef.PrimaryAggregationType)], metricDef) + } + } else { + supportedAggregations, unsupportedAggregations = filterAggregations(metricConfig.Aggregations, metricDefs) + if len(unsupportedAggregations) > 0 { + return nil, errors.Errorf("the aggregations configured : %s are not supported for some of the metrics selected %s ", + strings.Join(unsupportedAggregations, ","), strings.Join(metricNames, ",")) + } + if len(supportedAggregations) == 0 { + return nil, errors.Errorf("no aggregations were found based on the aggregation values configured or supported between the metrics : %s", + strings.Join(metricNames, ",")) + } + key := strings.Join(supportedAggregations, ",") + metricGroups[key] = append(metricGroups[key], metricDefs...) + } + return metricGroups, nil +} + // filterAggregations will filter out any unsupported aggregations based on the metrics selected func filterAggregations(selectedRange []string, metrics []insights.MetricDefinition) ([]string, []string) { var difference []string diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index 3115ac952b0d..06499e0be5a4 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -60,12 +60,13 @@ func TestMapMetric(t *testing.T) { Value: MockMetricDefinitions(), } metricConfig := azure.MetricConfig{Namespace: "namespace", Dimensions: []azure.DimensionConfig{{Name: "location", Value: "West Europe"}}} + resourceConfig := azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} client := azure.NewMockClient() t.Run("return error when no metric definitions were found", func(t *testing.T) { m := &azure.MockService{} m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(insights.MetricDefinitionCollection{}, errors.New("invalid resource ID")) client.AzureMonitorService = m - metric, err := mapMetric(client, metricConfig, resource) + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NotNil(t, err) assert.Equal(t, metric, []azure.Metric(nil)) m.AssertExpectations(t) @@ -75,7 +76,8 @@ func TestMapMetric(t *testing.T) { m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metricConfig.Name = []string{"*"} - metrics, err := mapMetric(client, metricConfig, resource) + resourceConfig.Metrics = []azure.MetricConfig{metricConfig} + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) assert.Equal(t, metrics[0].Resource.ID, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") @@ -93,7 +95,8 @@ func TestMapMetric(t *testing.T) { client.AzureMonitorService = m metricConfig.Name = []string{"TotalRequests", "Capacity"} metricConfig.Aggregations = []string{"Average"} - metrics, err := mapMetric(client, metricConfig, resource) + resourceConfig.Metrics = []azure.MetricConfig{metricConfig} + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) assert.True(t, len(metrics) > 0) diff --git a/x-pack/metricbeat/module/azure/monitor/monitor.go b/x-pack/metricbeat/module/azure/monitor/monitor.go index 70776baa13ac..6efea011fa4a 100644 --- a/x-pack/metricbeat/module/azure/monitor/monitor.go +++ b/x-pack/metricbeat/module/azure/monitor/monitor.go @@ -32,7 +32,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } - ms.MapMetric = mapMetric + ms.MapMetrics = mapMetrics return &MetricSet{ MetricSet: ms, }, nil diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index a9fcdf7bb3b5..da51accc5029 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -11,6 +11,8 @@ import ( // Resource will contain the main azure resource details type Resource struct { + // SubID is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms + SubID string ID string Name string Location string diff --git a/x-pack/metricbeat/module/azure/storage/_meta/data.json b/x-pack/metricbeat/module/azure/storage/_meta/data.json new file mode 100644 index 000000000000..908a5da9ae86 --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/_meta/data.json @@ -0,0 +1,84 @@ +{ + "_index" : "metricbeat-8.0.0-2020.01.09-000001", + "_type" : "_doc", + "_id" : "XxYjjG8BQe78yNOEwING", + "_score" : null, + "_source" : { + "@timestamp" : "2020-01-09T21:04:00.000Z", + "event" : { + "duration" : 122304420600, + "dataset" : "azure.storage", + "module" : "azure" + }, + "metricset" : { + "name" : "storage", + "period" : 300000 + }, + "azure" : { + "namespace" : "Microsoft.Storage/storageAccounts/blobServices", + "dimensions" : { + "apiname" : "DeleteBlob" + }, + "storage" : { + "successe2elatency" : { + "avg" : 9.1 + }, + "availability" : { + "avg" : 100 + }, + "transactions" : { + "total" : 10 + }, + "ingress" : { + "total" : 5330 + }, + "egress" : { + "total" : 1010 + }, + "successserverlatency" : { + "avg" : 9.1 + } + }, + "resource" : { + "name" : "obsdiaglinux", + "type" : "Microsoft.Storage/storageAccounts", + "group" : "obs-infrastructure" + }, + "subscription_id" : "12345678-qwer-1234-5678-12345678" + }, + "service" : { + "type" : "azure" + }, + "ecs" : { + "version" : "1.2.0" + }, + "host" : { + "architecture" : "x86_64", + "name" : "DESKTOP-RFOOE09", + "os" : { + "kernel" : "10.0.18362.535 (WinBuild.160101.0800)", + "build" : "18362.535", + "platform" : "windows", + "version" : "10.0", + "family" : "windows", + "name" : "Windows 10 Pro" + }, + "id" : "12345678-qwer-1234-5678-12345678", + "hostname" : "DESKTOP-RFOOE09" + }, + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "12345678-qwer-1234-5678-12345678", + "version" : "8.0.0", + "type" : "metricbeat", + "ephemeral_id" : "12345678-qwer-1234-5678-12345678" + }, + "cloud" : { + "provider" : "azure", + "region" : "westeurope" + } + }, + "sort" : [ + 1578603840000 + ] +} diff --git a/x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc new file mode 100644 index 000000000000..08701be90dff --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc @@ -0,0 +1,35 @@ +This is the storage metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified storage accounts. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list of storage accounts we want to apply our metric configuration options on. + +`service_type`:: (_[]string_) This configuration key can be used with any of the 2 options above, for example: + +---- +resources: + - resource_id: "" + service_type: ["blob", "table"] + - resource_group: "" + service_type: ["queue", "file"] + +---- + +it will filter the metric values to be returned by specific metric namespaces. The supported metrics and namespaces can be found here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftstoragestorageaccounts. +The service type values allowed are `blob`, `table`, `queue`, `file` based on the namespaces `Microsoft.Storage/storageAccounts/blobServices`,`Microsoft.Storage/storageAccounts/tableServices`,`Microsoft.Storage/storageAccounts/fileServices`,`Microsoft.Storage/storageAccounts/queueServices`. +If no service_type is specified all values are applied. + +Also, if the `resources` option is not specified, then all the storage accounts from the entire subscription will be selected. +The primary aggregation value will be retrieved for all the metrics contained in the namespaces. The aggregation options are `avg`, `sum`, `min`, `max`, `total`, `count`. + +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/storage/_meta/fields.yml b/x-pack/metricbeat/module/azure/storage/_meta/fields.yml new file mode 100644 index 000000000000..c103e77f8414 --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: storage.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + storage account diff --git a/x-pack/metricbeat/module/azure/storage/client_helper.go b/x-pack/metricbeat/module/azure/storage/client_helper.go new file mode 100644 index 000000000000..488f90d21ebd --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/client_helper.go @@ -0,0 +1,149 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package storage + +import ( + "fmt" + + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" + "github.com/pkg/errors" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +const resourceIDExtension = "/default" +const serviceTypeNamespaceExtension = "Services" + +// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters +func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { + var metrics []azure.Metric + // list all storage account namespaces for this metricset + namespaces := []string{defaultStorageAccountNamespace} + // if serviceType is configured, add only the selected serviceType namespaces + if len(resourceConfig.ServiceType) > 0 { + for _, selectedServiceNamespace := range resourceConfig.ServiceType { + namespaces = append(namespaces, fmt.Sprintf("%s/%s%s", defaultStorageAccountNamespace, selectedServiceNamespace, serviceTypeNamespaceExtension)) + } + } else { + for _, service := range storageServiceNamespaces { + namespaces = append(namespaces, fmt.Sprintf("%s%s", defaultStorageAccountNamespace, service)) + } + } + for _, resource := range resources { + for _, namespace := range namespaces { + // resourceID will be different for a serviceType namespace, format will be resourceID/service/default + var resourceID = *resource.ID + if i := retrieveServiceNamespace(namespace); i != "" { + resourceID += i + resourceIDExtension + } + // get all metric definitions supported by the namespace provided + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(resourceID, namespace) + if err != nil { + return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", resourceID, namespace) + } + if len(*metricDefinitions.Value) == 0 { + return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", resourceID, namespace) + } + var filteredMetricDefinitions []insights.MetricDefinition + for _, metricDefinition := range *metricDefinitions.Value { + filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) + } + // some metrics do not support the default PT5M timegrain so they will have to be grouped in a different API call, else call will fail + groupedMetrics := groupOnTimeGrain(filteredMetricDefinitions) + for time, groupedMetricList := range groupedMetrics { + // metrics will have to be grouped by allowed dimensions + dimMetrics := groupMetricsByAllowedDimensions(groupedMetricList) + for dimension, mets := range dimMetrics { + var dimensions []azure.Dimension + if dimension != azure.NoDimension { + dimensions = []azure.Dimension{{Name: dimension, Value: "*"}} + } + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, mets, resource, resourceID, namespace, dimensions, time)...) + } + } + } + } + return metrics, nil +} + +// groupOnTimeGrain - some metrics do not support the default timegrain value so the closest supported timegrain will be selected +func groupOnTimeGrain(list []insights.MetricDefinition) map[string][]insights.MetricDefinition { + var groupedList = make(map[string][]insights.MetricDefinition) + for _, metric := range list { + timegrain := retrieveSupportedMetricAvailability(*metric.MetricAvailabilities) + if _, ok := groupedList[timegrain]; !ok { + groupedList[timegrain] = make([]insights.MetricDefinition, 0) + } + groupedList[timegrain] = append(groupedList[timegrain], metric) + } + return groupedList +} + +// retrieveSupportedMetricAvailability func will return the default timegrain if supported, else will return the next timegrain +func retrieveSupportedMetricAvailability(availabilities []insights.MetricAvailability) string { + // common case in metrics supported by storage account - one availability + if len(availabilities) == 1 { + return *availabilities[0].TimeGrain + } + // check if the default timegrain is supported + for _, availability := range availabilities { + if *availability.TimeGrain == azure.DefaultTimeGrain { + return azure.DefaultTimeGrain + } + } + // select first timegrain, should be bigger than the min timegrain of 1M, timegrains are returned in asc order + if *availabilities[0].TimeGrain != "PT1M" { + return *availabilities[0].TimeGrain + } + return *availabilities[1].TimeGrain +} + +// retrieveServiceNamespace func will check if the namespace is part of the service namespaces and returns the the selected name +func retrieveServiceNamespace(item string) string { + for _, i := range storageServiceNamespaces { + if defaultStorageAccountNamespace+i == item { + return i + } + } + return "" +} + +// filterAllowedDimension func will filter out all unallowed dimensions +func filterAllowedDimension(metric insights.MetricDefinition) []string { + if metric.Dimensions == nil { + return nil + } + var dimensions []string + for _, dimension := range *metric.Dimensions { + for _, dim := range allowedDimensions { + if dim == *dimension.Value { + dimensions = append(dimensions, dim) + } + } + } + return dimensions +} + +// groupMetricsByAllowedDimensions will group metrics by dimension names in order to reduce the number of api calls +func groupMetricsByAllowedDimensions(metrics []insights.MetricDefinition) map[string][]insights.MetricDefinition { + var groupedMetrics = make(map[string][]insights.MetricDefinition) + for _, metric := range metrics { + if dimensions := filterAllowedDimension(metric); len(dimensions) > 0 { + for _, dimension := range dimensions { + if _, ok := groupedMetrics[dimension]; !ok { + groupedMetrics[dimension] = make([]insights.MetricDefinition, 0) + } + groupedMetrics[dimension] = append(groupedMetrics[dimension], metric) + } + } else { + if _, ok := groupedMetrics[azure.NoDimension]; !ok { + groupedMetrics[azure.NoDimension] = make([]insights.MetricDefinition, 0) + } + groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metric) + } + } + return groupedMetrics +} diff --git a/x-pack/metricbeat/module/azure/storage/client_helper_test.go b/x-pack/metricbeat/module/azure/storage/client_helper_test.go new file mode 100644 index 000000000000..f9dd066df33e --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/client_helper_test.go @@ -0,0 +1,180 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package storage + +import ( + "reflect" + "testing" + + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +var ( + time1 = "PT1M" + time2 = "PT5M" + time3 = "PT1H" + availability1 = []insights.MetricAvailability{ + {TimeGrain: &time1}, + {TimeGrain: &time2}, + } + availability2 = []insights.MetricAvailability{ + {TimeGrain: &time3}, + } + availability3 = []insights.MetricAvailability{ + {TimeGrain: &time1}, + {TimeGrain: &time3}, + } +) + +func MockResource() resources.GenericResource { + id := "123" + name := "resourceName" + location := "resourceLocation" + rType := "resourceType" + return resources.GenericResource{ + ID: &id, + Name: &name, + Location: &location, + Type: &rType, + } +} + +func MockNamespace() insights.MetricNamespaceCollection { + name := "namespace" + property := insights.MetricNamespaceName{ + MetricNamespaceName: &name, + } + namespace := insights.MetricNamespace{ + Name: &name, + Properties: &property, + } + list := []insights.MetricNamespace{namespace} + return insights.MetricNamespaceCollection{ + Value: &list, + } +} + +func MockMetricDefinitions() *[]insights.MetricDefinition { + metric1 := "TotalRequests" + metric2 := "Capacity" + defs := []insights.MetricDefinition{ + { + Name: &insights.LocalizableString{Value: &metric1}, + PrimaryAggregationType: insights.Average, + MetricAvailabilities: &availability1, + SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, + }, + { + Name: &insights.LocalizableString{Value: &metric2}, + PrimaryAggregationType: insights.Average, + MetricAvailabilities: &availability2, + SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, + }, + } + return &defs +} + +func TestMapMetric(t *testing.T) { + resource := MockResource() + metricDefinitions := insights.MetricDefinitionCollection{ + Value: MockMetricDefinitions(), + } + emptyList := []insights.MetricDefinition{} + emptyMetricDefinitions := insights.MetricDefinitionCollection{ + Value: &emptyList, + } + metricConfig := azure.MetricConfig{Name: []string{"*"}} + resourceConfig := azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}, ServiceType: []string{"blob"}} + client := azure.NewMockClient() + t.Run("return error when no metric definitions were found", func(t *testing.T) { + m := &azure.MockService{} + m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) + client.AzureMonitorService = m + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) + assert.NotNil(t, err) + assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace Microsoft.Storage/storageAccounts.") + assert.Equal(t, metric, []azure.Metric(nil)) + m.AssertExpectations(t) + }) + t.Run("return mapped metrics correctly", func(t *testing.T) { + m := &azure.MockService{} + m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) + client.AzureMonitorService = m + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) + assert.Nil(t, err) + assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Name, "resourceName") + assert.Equal(t, metrics[0].Resource.Type, "resourceType") + assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].Namespace, "Microsoft.Storage/storageAccounts") + assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Name, "resourceName") + assert.Equal(t, metrics[1].Resource.Type, "resourceType") + assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Namespace, "Microsoft.Storage/storageAccounts") + assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) + assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) + + //order of elements can be different when running the test + assert.Equal(t, len(metrics), 4) + for _, metricValue := range metrics { + assert.Equal(t, metricValue.Aggregations, "Average") + assert.Equal(t, len(metricValue.Names), 1) + assert.Contains(t, []string{"TotalRequests", "Capacity"}, metricValue.Names[0]) + if reflect.DeepEqual(metricValue.Names, []string{"Capacity"}) { + assert.Equal(t, metricValue.TimeGrain, "PT1H") + } else { + assert.Equal(t, metricValue.TimeGrain, "PT5M") + } + } + m.AssertExpectations(t) + }) +} + +func TestFilterOnTimeGrain(t *testing.T) { + var list = []insights.MetricDefinition{ + {MetricAvailabilities: &availability1}, + {MetricAvailabilities: &availability2}, + {MetricAvailabilities: &availability3}, + } + response := groupOnTimeGrain(list) + assert.Equal(t, len(response), 2) + result := [][]insights.MetricDefinition{ + { + {MetricAvailabilities: &availability1}, + }, + { + {MetricAvailabilities: &availability2}, + {MetricAvailabilities: &availability3}, + }, + } + for key, availabilities := range response { + assert.Contains(t, []string{time2, time3}, key) + assert.Contains(t, result, availabilities) + } +} + +func TestRetrieveSupportedMetricAvailability(t *testing.T) { + response := retrieveSupportedMetricAvailability(availability1) + assert.Equal(t, response, time2) + response = retrieveSupportedMetricAvailability(availability2) + assert.Equal(t, response, time3) + response = retrieveSupportedMetricAvailability(availability3) + assert.Equal(t, response, time3) +} + +func TestRetrieveServiceNamespace(t *testing.T) { + var test = "Microsoft.Storage/storageAccounts/tableServices" + response := retrieveServiceNamespace(test) + assert.Equal(t, response, "/tableServices") + test = "Microsoft.Storage/storageAccounts" + response = retrieveServiceNamespace(test) + assert.Equal(t, response, "") +} diff --git a/x-pack/metricbeat/module/azure/storage/storage.go b/x-pack/metricbeat/module/azure/storage/storage.go new file mode 100644 index 000000000000..9a386b82519b --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/storage.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package storage + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +const defaultStorageAccountNamespace = "Microsoft.Storage/storageAccounts" + +var ( + storageServiceNamespaces = []string{"/blobServices", "/tableServices", "/queueServices", "/fileServices"} + allowedDimensions = []string{"ResponseType", "ApiName"} +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("azure", "storage", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *azure.MetricSet +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := azure.NewMetricSet(base) + if err != nil { + return nil, err + } + // if no options are entered we will retrieve all the vm's from the entire subscription + if len(ms.Client.Config.Resources) == 0 { + ms.Client.Config.Resources = []azure.ResourceConfig{ + { + Query: fmt.Sprintf("resourceType eq '%s'", defaultStorageAccountNamespace), + }, + } + } + for index := range ms.Client.Config.Resources { + // if any resource groups were configured the resource type should be added + if len(ms.Client.Config.Resources[index].Group) > 0 { + ms.Client.Config.Resources[index].Type = defaultStorageAccountNamespace + } + // one metric configuration will be added containing all metrics names + ms.Client.Config.Resources[index].Metrics = []azure.MetricConfig{ + { + Name: []string{"*"}, + }, + } + } + ms.MapMetrics = mapMetrics + return &MetricSet{ + MetricSet: ms, + }, nil +} diff --git a/x-pack/metricbeat/module/azure/storage/storage_test.go b/x-pack/metricbeat/module/azure/storage/storage_test.go new file mode 100644 index 000000000000..6add3aae4584 --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/storage_test.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package storage + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +var ( + missingResourcesConfig = common.MapStr{ + "module": "azure", + "period": "60s", + "metricsets": []string{"storage"}, + "client_secret": "unique identifier", + "client_id": "unique identifier", + "subscription_id": "unique identifier", + "tenant_id": "unique identifier", + } + + resourceConfig = common.MapStr{ + "module": "azure", + "period": "60s", + "metricsets": []string{"storage"}, + "client_secret": "unique identifier", + "client_id": "unique identifier", + "subscription_id": "unique identifier", + "tenant_id": "unique identifier", + "resources": []common.MapStr{ + { + "resource_id": "test", + "metrics": []map[string]interface{}{ + { + "name": []string{"*"}, + }}, + }}, + } +) + +func TestFetch(t *testing.T) { + c, err := common.NewConfigFrom(missingResourcesConfig) + if err != nil { + t.Fatal(err) + } + module, metricsets, err := mb.NewModule(c, mb.Registry) + assert.NotNil(t, module) + assert.NotNil(t, metricsets) + assert.Nil(t, err) + ms, ok := metricsets[0].(*MetricSet) + assert.Equal(t, len(ms.Client.Config.Resources), 1) + assert.Equal(t, ms.Client.Config.Resources[0].Query, fmt.Sprintf("resourceType eq '%s'", defaultStorageAccountNamespace)) + + c, err = common.NewConfigFrom(resourceConfig) + if err != nil { + t.Fatal(err) + } + module, metricsets, err = mb.NewModule(c, mb.Registry) + assert.NotNil(t, module) + assert.NotNil(t, metricsets) + ms, ok = metricsets[0].(*MetricSet) + require.True(t, ok, "metricset must be MetricSet") + assert.NotNil(t, ms) +} diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 8f10f2d43fd2..31cb3a189510 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -38,3 +38,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - storage +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s