Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[gcp/billing] always quote table name identifier #26870

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 30 additions & 17 deletions x-pack/metricbeat/module/gcp/billing/billing.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,26 +205,12 @@ func getTables(ctx context.Context, client *bigquery.Client, datasetID string, t

func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, tableMeta tableMeta, month string, costType string) ([]mb.Event, error) {
var events []mb.Event
query := fmt.Sprintf(`
SELECT
invoice.month,
project.id,
project.name,
billing_account_id,
cost_type,
(SUM(CAST(cost * 1000000 AS int64))
+ SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000
AS total_exact
FROM %s
WHERE project.id IS NOT NULL
AND invoice.month = '%s'
AND cost_type = '%s'
GROUP BY 1, 2, 3, 4, 5
ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, tableMeta.tableFullID, month, costType)

q := client.Query(query)
query := generateQuery(tableMeta.tableFullID, month, costType)
m.logger.Debug("bigquery query = ", query)

q := client.Query(query)

// Location must match that of the dataset(s) referenced in the query.
q.Location = tableMeta.location

Expand Down Expand Up @@ -308,3 +294,30 @@ func generateEventID(currentDate string, rowItems []bigquery.Value) string {
prefix := hex.EncodeToString(h.Sum(nil))
return prefix[:20]
}

// generateQuery returns the query to be used by the BigQuery client to retrieve monthly
// cost types breakdown.
func generateQuery(tableName, month, costType string) string {
// The table name is user provided, so it may contains special characters.
// In order to allow any character in the table identifier, use the Quoted identifier format.
// See https://github.com/elastic/beats/issues/26855
// NOTE: is not possible to escape backtics (`) in a multiline string
escapedTableName := fmt.Sprintf("`%s`", tableName)
query := fmt.Sprintf(`
SELECT
invoice.month,
project.id,
project.name,
billing_account_id,
cost_type,
(SUM(CAST(cost * 1000000 AS int64))
+ SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000
AS total_exact
FROM %s
WHERE project.id IS NOT NULL
AND invoice.month = '%s'
AND cost_type = '%s'
GROUP BY 1, 2, 3, 4, 5
ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, escapedTableName, month, costType)
return query
}
16 changes: 16 additions & 0 deletions x-pack/metricbeat/module/gcp/billing/billing_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package billing

import (
"io/ioutil"
"log"
"strconv"
"testing"

Expand All @@ -16,3 +18,17 @@ func TestGetCurrentMonth(t *testing.T) {
_, err := strconv.ParseInt(currentMonth, 0, 64)
assert.NoError(t, err)
}

func TestGenerateQuery(t *testing.T) {
log.SetOutput(ioutil.Discard)

query := generateQuery("my-table", "jan", "cost")
log.Println(query)

// verify that table name quoting is in effect
assert.Contains(t, query, "`my-table`")
// verify the group by is preserved
assert.Contains(t, query, "GROUP BY 1, 2, 3, 4, 5")
// verify the order by is preserved
assert.Contains(t, query, "ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC")
}