From b88364f3c7f6ed1948fcee912b503d320aadefb1 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 22 Jun 2021 14:36:26 +0000 Subject: [PATCH 1/5] #26357: Change Account ID to Project ID --- x-pack/metricbeat/module/gcp/billing/billing.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 13236ff4ae4..a5f29315cb0 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -268,7 +268,7 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, return events, nil } -func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { +func createEvents(rowItems []bigquery.Value, projectID string) mb.Event { event := mb.Event{} event.MetricSetFields = common.MapStr{ "invoice_month": rowItems[0], @@ -279,8 +279,8 @@ func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { event.RootFields = common.MapStr{ "cloud.provider": "gcp", - "cloud.account.id": accountID, - "cloud.account.name": accountID, + "cloud.project.id": projectID, + "cloud.project.name": projectID, } // create eventID for each current_date + invoice_month + project_id + cost_type From 23b51d7963442d0535d00c488d04af2e31691d08 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 22 Jun 2021 15:11:02 +0000 Subject: [PATCH 2/5] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9ba9149e252..18c8505acbc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -491,6 +491,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327] - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] - Major refactor of system/cpu and system/core metrics. {pull}25771[25771] +- Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412] *Packetbeat* From 5f2f9bffe3ae1c8b1c6609ed23abc92497c98a62 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 23 Jun 2021 11:55:41 +0000 Subject: [PATCH 3/5] query additional items --- x-pack/metricbeat/module/gcp/billing/billing.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index a5f29315cb0..4077bc5f588 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -209,6 +209,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, SELECT invoice.month, project.id, + project.name, + billing_account_id, cost_type, (SUM(CAST(cost * 1000000 AS int64)) + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 @@ -261,7 +263,7 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, return events, err } - if len(row) == 4 { + if len(row) == 6 { events = append(events, createEvents(row, m.config.ProjectID)) } } @@ -271,16 +273,19 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, func createEvents(rowItems []bigquery.Value, projectID string) mb.Event { event := mb.Event{} event.MetricSetFields = common.MapStr{ - "invoice_month": rowItems[0], - "project_id": rowItems[1], - "cost_type": rowItems[2], - "total": rowItems[3], + "invoice_month": rowItems[0], + "project_id": rowItems[1], + "project_name": rowItems[3], + "billing_account_id": rowItems[4], + "cost_type": rowItems[5], + "total": rowItems[9], } event.RootFields = common.MapStr{ "cloud.provider": "gcp", "cloud.project.id": projectID, - "cloud.project.name": projectID, + "cloud.project.name": rowItems[3], + "cloud.account.id": rowItems[4], } // create eventID for each current_date + invoice_month + project_id + cost_type From b62ff8074b8ed26ca95174c1ecff688edf26b06f Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 24 Jun 2021 11:41:42 +0000 Subject: [PATCH 4/5] fix typo --- x-pack/metricbeat/module/gcp/billing/billing.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 4077bc5f588..4c2cc953671 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -275,17 +275,17 @@ func createEvents(rowItems []bigquery.Value, projectID string) mb.Event { event.MetricSetFields = common.MapStr{ "invoice_month": rowItems[0], "project_id": rowItems[1], - "project_name": rowItems[3], - "billing_account_id": rowItems[4], - "cost_type": rowItems[5], - "total": rowItems[9], + "project_name": rowItems[2], + "billing_account_id": rowItems[3], + "cost_type": rowItems[4], + "total": rowItems[5], } event.RootFields = common.MapStr{ "cloud.provider": "gcp", "cloud.project.id": projectID, - "cloud.project.name": rowItems[3], - "cloud.account.id": rowItems[4], + "cloud.project.name": rowItems[2], + "cloud.account.id": rowItems[3], } // create eventID for each current_date + invoice_month + project_id + cost_type From 2352c7f6a7b371cdbf9fd49666ab628d6eb2052b Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 24 Jun 2021 23:12:09 +0000 Subject: [PATCH 5/5] fix Big Query per comments --- x-pack/metricbeat/module/gcp/billing/billing.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 4c2cc953671..a14d263ac6b 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -219,8 +219,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, WHERE project.id IS NOT NULL AND invoice.month = '%s' AND cost_type = '%s' - GROUP BY 1, 2, 3 - ORDER BY 1 ASC, 2 ASC, 3 ASC;`, tableMeta.tableFullID, month, costType) + GROUP BY 1, 2, 3, 4, 5 + ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, tableMeta.tableFullID, month, costType) q := client.Query(query) m.logger.Debug("bigquery query = ", query)