This repository has been archived by the owner on Jan 19, 2023. It is now read-only.
generated from vshn/go-bootstrap
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #4 from vshn/accumulate-first
Accumulate data first & fix error handling
- Loading branch information
Showing
6 changed files
with
285 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,5 +12,8 @@ cloudscale-metrics-collector | |
| .public/ | ||
| node_modules/ | ||
|
|
||
| # IDE | ||
| .idea/ | ||
|
|
||
| # Configuration | ||
| env | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/cloudscale-ch/cloudscale-go-sdk/v2" | ||
| "os" | ||
| "time" | ||
| ) | ||
|
|
||
| // AccumulateKey represents one data point ("fact") in the billing database. | ||
| // The actual value for the data point is not present in this type, as this type is just a map key, and the corresponding value is stored as a map value. | ||
| type AccumulateKey struct { | ||
| Query string | ||
| Zone string | ||
| Tenant string | ||
| Namespace string | ||
| Start time.Time | ||
| } | ||
|
|
||
| // String returns the full "source" string as used by the appuio-cloud-reporting | ||
| func (this AccumulateKey) String() string { | ||
| return this.Query + ":" + this.Zone + ":" + this.Tenant + ":" + this.Namespace | ||
| } | ||
|
|
||
| /* | ||
| accumulateBucketMetrics gets all the bucket metrics from cloudscale and puts them into a map. The map key is the "AccumulateKey", | ||
| and the value is the raw value of the data returned by cloudscale (e.g. bytes, requests). In order to construct the | ||
| correct AccumulateKey, this function needs to fetch the ObjectUsers's tags, because that's where the zone, tenant and | ||
| namespace are stored. | ||
| This method is "accumulating" data because it collects data from possibly multiple ObjectsUsers under the same | ||
| AccumulateKey. This is because the billing system can't handle multiple ObjectsUsers per namespace. | ||
| */ | ||
| func accumulateBucketMetrics(ctx context.Context, date time.Time, cloudscaleClient *cloudscale.Client) (map[AccumulateKey]uint64, error) { | ||
| bucketMetricsRequest := cloudscale.BucketMetricsRequest{Start: date, End: date} | ||
| bucketMetrics, err := cloudscaleClient.Metrics.GetBucketMetrics(ctx, &bucketMetricsRequest) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| accumulated := make(map[AccumulateKey]uint64) | ||
|
|
||
| for _, bucketMetricsData := range bucketMetrics.Data { | ||
| objectsUser, err := cloudscaleClient.ObjectsUsers.Get(ctx, bucketMetricsData.Subject.ObjectsUserID) | ||
| if err != nil || objectsUser == nil { | ||
| fmt.Fprintf(os.Stderr, "WARNING: Cannot sync bucket %s, objects user %s not found\n", bucketMetricsData.Subject.BucketName, bucketMetricsData.Subject.ObjectsUserID) | ||
| continue | ||
| } | ||
| err = accumulateBucketMetricsForObjectsUser(accumulated, bucketMetricsData, objectsUser) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "WARNING: Cannot sync bucket %s: %v\n", bucketMetricsData.Subject.BucketName, err) | ||
| continue | ||
| } | ||
| } | ||
|
|
||
| return accumulated, nil | ||
| } | ||
|
|
||
| func accumulateBucketMetricsForObjectsUser(accumulated map[AccumulateKey]uint64, bucketMetricsData cloudscale.BucketMetricsData, objectsUser *cloudscale.ObjectsUser) error { | ||
| if len(bucketMetricsData.TimeSeries) != 1 { | ||
| return fmt.Errorf("There must be exactly one metrics data point, found %d", len(bucketMetricsData.TimeSeries)) | ||
| } | ||
|
|
||
| tenantStr := objectsUser.Tags["tenant"] | ||
| if tenantStr == "" { | ||
| return fmt.Errorf("no tenant information found on objectsUser") | ||
| } | ||
| namespace := objectsUser.Tags["namespace"] | ||
| if namespace == "" { | ||
| return fmt.Errorf("no namespace information found on objectsUser") | ||
| } | ||
| zone := objectsUser.Tags["zone"] | ||
| if zone == "" { | ||
| return fmt.Errorf("no zone information found on objectsUser") | ||
| } | ||
|
|
||
| sourceStorage := AccumulateKey{ | ||
| Query: sourceQueryStorage, | ||
| Zone: zone, | ||
| Tenant: tenantStr, | ||
| Namespace: namespace, | ||
| Start: bucketMetricsData.TimeSeries[0].Start, | ||
| } | ||
| sourceTrafficOut := AccumulateKey{ | ||
| Query: sourceQueryTrafficOut, | ||
| Zone: zone, | ||
| Tenant: tenantStr, | ||
| Namespace: namespace, | ||
| Start: bucketMetricsData.TimeSeries[0].Start, | ||
| } | ||
| sourceRequests := AccumulateKey{ | ||
| Query: sourceQueryRequests, | ||
| Zone: zone, | ||
| Tenant: tenantStr, | ||
| Namespace: namespace, | ||
| Start: bucketMetricsData.TimeSeries[0].Start, | ||
| } | ||
|
|
||
| accumulated[sourceStorage] += uint64(bucketMetricsData.TimeSeries[0].Usage.StorageBytes) | ||
| accumulated[sourceTrafficOut] += uint64(bucketMetricsData.TimeSeries[0].Usage.SentBytes) | ||
| accumulated[sourceRequests] += uint64(bucketMetricsData.TimeSeries[0].Usage.Requests) | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/cloudscale-ch/cloudscale-go-sdk/v2" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| // assertEqualfUint64 implements the functionality of assert.Equalf for uint64, because assert.Equalf cannot print uint64 correctly. | ||
| // See https://github.com/stretchr/testify/issues/400 | ||
| func assertEqualfUint64(t *testing.T, expected uint64, actual uint64, msg string, args ...interface{}) bool { | ||
| if expected != actual { | ||
| return assert.Fail(t, fmt.Sprintf("Not equal: \n"+ | ||
| "expected: %d\n"+ | ||
| "actual : %d", expected, actual)) | ||
| } | ||
| return true | ||
| } | ||
|
|
||
| func TestAccumulateBucketMetricsForObjectsUser(t *testing.T) { | ||
| zone := "appuio-cloudscale-ch-lpg" | ||
| tenant := "inity" | ||
| namespace := "testnamespace" | ||
|
|
||
| // get the correct date for a data set that was created yesterday | ||
| location, err := time.LoadLocation("Europe/Zurich") | ||
| require.NoError(t, err, "could not load location Europe/Zurich") | ||
| now := time.Now().In(location) | ||
| date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location()) | ||
|
|
||
| // build input data structure | ||
| bucketMetricsInterval := []cloudscale.BucketMetricsInterval{ | ||
| { | ||
| Start: date, | ||
| End: date, | ||
| Usage: cloudscale.BucketMetricsIntervalUsage{ | ||
| Requests: 5, | ||
| StorageBytes: 1000000, | ||
| SentBytes: 2000000, | ||
| }, | ||
| }, | ||
| } | ||
| bucketMetricsData := cloudscale.BucketMetricsData{ | ||
| TimeSeries: bucketMetricsInterval, | ||
| } | ||
| objectsUser := cloudscale.ObjectsUser{} | ||
| objectsUser.Tags = cloudscale.TagMap{"zone": zone, "tenant": tenant, "namespace": namespace} | ||
|
|
||
| accumulated := make(map[AccumulateKey]uint64) | ||
| accumulateBucketMetricsForObjectsUser(accumulated, bucketMetricsData, &objectsUser) | ||
|
|
||
| require.Len(t, accumulated, 3, "incorrect amount of values 'accumulated'") | ||
|
|
||
| key := AccumulateKey{ | ||
| Zone: zone, | ||
| Tenant: tenant, | ||
| Namespace: namespace, | ||
| Start: date, | ||
| } | ||
|
|
||
| key.Query = "object-storage-requests" | ||
| assertEqualfUint64(t, uint64(5), accumulated[key], "incorrect value in %s", key) | ||
|
|
||
| key.Query = "object-storage-storage" | ||
| assertEqualfUint64(t, uint64(1000000), accumulated[key], "incorrect value in %s", key) | ||
|
|
||
| key.Query = "object-storage-traffic-out" | ||
| assertEqualfUint64(t, uint64(2000000), accumulated[key], "incorrect value in %s", key) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.