diff --git a/dart/lib/src/metrics/metric.dart b/dart/lib/src/metrics/metric.dart index 52831e73ce..fdea81cbf4 100644 --- a/dart/lib/src/metrics/metric.dart +++ b/dart/lib/src/metrics/metric.dart @@ -4,10 +4,9 @@ import 'package:meta/meta.dart'; import '../../sentry.dart'; -final RegExp forbiddenKeyCharsRegex = RegExp('[^a-zA-Z0-9_/.-]+'); -final RegExp forbiddenValueCharsRegex = - RegExp('[^\\w\\d\\s_:/@\\.\\{\\}\\[\\]\$-]+'); -final RegExp forbiddenUnitCharsRegex = RegExp('[^a-zA-Z0-9_/.]+'); +final RegExp unitRegex = RegExp('[^\\w]+'); +final RegExp nameRegex = RegExp('[^\\w-.]+'); +final RegExp tagKeyRegex = RegExp('[^\\w-./]+'); /// Base class for metrics. /// Each metric is identified by a [key]. Its [type] describes its behaviour. @@ -69,7 +68,7 @@ abstract class Metric { /// and it's appended at the end of the encoded metric. String encodeToStatsd(int bucketKey) { final buffer = StringBuffer(); - buffer.write(_normalizeKey(key)); + buffer.write(_sanitizeName(key)); buffer.write("@"); final sanitizeUnitName = _sanitizeUnit(unit.name); @@ -87,7 +86,7 @@ abstract class Metric { buffer.write("|#"); final serializedTags = tags.entries .map((tag) => - '${_normalizeKey(tag.key)}:${_normalizeTagValue(tag.value)}') + '${_sanitizeTagKey(tag.key)}:${_sanitizeTagValue(tag.value)}') .join(','); buffer.write(serializedTags); } @@ -117,16 +116,43 @@ abstract class Metric { String getSpanAggregationKey() => '${type.statsdType}:$key@${unit.name}'; /// Remove forbidden characters from the metric key and tag key. - String _normalizeKey(String input) => - input.replaceAll(forbiddenKeyCharsRegex, '_'); + String _sanitizeName(String input) => input.replaceAll(nameRegex, '_'); /// Remove forbidden characters from the tag value. - String _normalizeTagValue(String input) => - input.replaceAll(forbiddenValueCharsRegex, ''); + String _sanitizeTagKey(String input) => input.replaceAll(tagKeyRegex, ''); /// Remove forbidden characters from the metric unit. - String _sanitizeUnit(String input) => - input.replaceAll(forbiddenUnitCharsRegex, '_'); + String _sanitizeUnit(String input) => input.replaceAll(unitRegex, ''); + + String _sanitizeTagValue(String input) { + // see https://develop.sentry.dev/sdk/metrics/#tag-values-replacement-map + // Line feed -> \n + // Carriage return -> \r + // Tab -> \t + // Backslash -> \\ + // Pipe -> \\u{7c} + // Comma -> \\u{2c} + final buffer = StringBuffer(); + for (int i = 0; i < input.length; i++) { + final ch = input[i]; + if (ch == '\n') { + buffer.write("\\n"); + } else if (ch == '\r') { + buffer.write("\\r"); + } else if (ch == '\t') { + buffer.write("\\t"); + } else if (ch == '\\') { + buffer.write("\\\\"); + } else if (ch == '|') { + buffer.write("\\u{7c}"); + } else if (ch == ',') { + buffer.write("\\u{2c}"); + } else { + buffer.write(ch); + } + } + return buffer.toString(); + } } /// Metric [MetricType.counter] that tracks a value that can only be incremented. diff --git a/dart/lib/src/transport/data_category.dart b/dart/lib/src/transport/data_category.dart index 1843acfacb..f21da18ed7 100644 --- a/dart/lib/src/transport/data_category.dart +++ b/dart/lib/src/transport/data_category.dart @@ -7,6 +7,7 @@ enum DataCategory { transaction, attachment, security, + metricBucket, unknown } @@ -27,6 +28,8 @@ extension DataCategoryExtension on DataCategory { return DataCategory.attachment; case 'security': return DataCategory.security; + case 'metric_bucket': + return DataCategory.metricBucket; } return DataCategory.unknown; } @@ -47,6 +50,8 @@ extension DataCategoryExtension on DataCategory { return 'attachment'; case DataCategory.security: return 'security'; + case DataCategory.metricBucket: + return 'metric_bucket'; case DataCategory.unknown: return 'unknown'; } diff --git a/dart/lib/src/transport/rate_limit.dart b/dart/lib/src/transport/rate_limit.dart index 8f41b91d81..00284a3ba7 100644 --- a/dart/lib/src/transport/rate_limit.dart +++ b/dart/lib/src/transport/rate_limit.dart @@ -2,8 +2,10 @@ import 'data_category.dart'; /// `RateLimit` containing limited `DataCategory` and duration in milliseconds. class RateLimit { - RateLimit(this.category, this.duration); + RateLimit(this.category, this.duration, {List? namespaces}) + : namespaces = (namespaces?..removeWhere((e) => e.isEmpty)) ?? []; final DataCategory category; final Duration duration; + final List namespaces; } diff --git a/dart/lib/src/transport/rate_limit_parser.dart b/dart/lib/src/transport/rate_limit_parser.dart index 63f4f179d1..03391e0210 100644 --- a/dart/lib/src/transport/rate_limit_parser.dart +++ b/dart/lib/src/transport/rate_limit_parser.dart @@ -14,6 +14,7 @@ class RateLimitParser { if (rateLimitHeader == null) { return []; } + // example: 2700:metric_bucket:organization:quota_exceeded:custom,... final rateLimits = []; final rateLimitValues = rateLimitHeader.toLowerCase().split(','); for (final rateLimitValue in rateLimitValues) { @@ -30,7 +31,17 @@ class RateLimitParser { final categoryValues = allCategories.split(';'); for (final categoryValue in categoryValues) { final category = DataCategoryExtension.fromStringValue(categoryValue); - if (category != DataCategory.unknown) { + // Metric buckets rate limit can have namespaces + if (category == DataCategory.metricBucket) { + final namespaces = durationAndCategories.length > 4 + ? durationAndCategories[4] + : null; + rateLimits.add(RateLimit( + category, + duration, + namespaces: namespaces?.trim().split(','), + )); + } else if (category != DataCategory.unknown) { rateLimits.add(RateLimit(category, duration)); } } diff --git a/dart/lib/src/transport/rate_limiter.dart b/dart/lib/src/transport/rate_limiter.dart index 6d4d3c3e9a..ef9b168edd 100644 --- a/dart/lib/src/transport/rate_limiter.dart +++ b/dart/lib/src/transport/rate_limiter.dart @@ -64,6 +64,11 @@ class RateLimiter { } for (final rateLimit in rateLimits) { + if (rateLimit.category == DataCategory.metricBucket && + rateLimit.namespaces.isNotEmpty && + !rateLimit.namespaces.contains('custom')) { + continue; + } _applyRetryAfterOnlyIfLonger( rateLimit.category, DateTime.fromMillisecondsSinceEpoch( @@ -111,6 +116,10 @@ class RateLimiter { return DataCategory.attachment; case 'transaction': return DataCategory.transaction; + // The envelope item type used for metrics is statsd, + // whereas the client report category is metric_bucket + case 'statsd': + return DataCategory.metricBucket; default: return DataCategory.unknown; } diff --git a/dart/test/metrics/metric_test.dart b/dart/test/metrics/metric_test.dart index d123916eea..f3edc0486b 100644 --- a/dart/test/metrics/metric_test.dart +++ b/dart/test/metrics/metric_test.dart @@ -58,10 +58,50 @@ void main() { test('encode CounterMetric', () async { final int bucketKey = 10; final expectedStatsd = - 'key_metric_@hour:2.1|c|#tag1:tag value 1,key_2:@13/-d_s|T10'; + 'key_metric_@hour:2.1|c|#tag1:tag\\u{2c} value 1,key2:&@"13/-d_s|T10'; final actualStatsd = fixture.counterMetric.encodeToStatsd(bucketKey); expect(actualStatsd, expectedStatsd); }); + + test('sanitize name', () async { + final metric = Metric.fromType( + type: MetricType.counter, + value: 2.1, + key: 'key£ - @# metric!', + unit: DurationSentryMeasurementUnit.day, + tags: {}, + ); + + final expectedStatsd = 'key_-_metric_@day:2.1|c|T10'; + expect(metric.encodeToStatsd(10), expectedStatsd); + }); + + test('sanitize unit', () async { + final metric = Metric.fromType( + type: MetricType.counter, + value: 2.1, + key: 'key', + unit: CustomSentryMeasurementUnit('weird-measurement name!'), + tags: {}, + ); + + final expectedStatsd = 'key@weirdmeasurementname:2.1|c|T10'; + expect(metric.encodeToStatsd(10), expectedStatsd); + }); + + test('sanitize tags', () async { + final metric = Metric.fromType( + type: MetricType.counter, + value: 2.1, + key: 'key', + unit: DurationSentryMeasurementUnit.day, + tags: {'tag1': 'tag, value 1', 'key 2': '&@"13/-d_s'}, + ); + + final expectedStatsd = + 'key@day:2.1|c|#tag1:tag\\u{2c} value 1,key2:&@"13/-d_s|T10'; + expect(metric.encodeToStatsd(10), expectedStatsd); + }); }); group('getCompositeKey', () { diff --git a/dart/test/protocol/rate_limit_parser_test.dart b/dart/test/protocol/rate_limit_parser_test.dart index c898915e04..567dec34f0 100644 --- a/dart/test/protocol/rate_limit_parser_test.dart +++ b/dart/test/protocol/rate_limit_parser_test.dart @@ -75,6 +75,45 @@ void main() { expect(sut[0].duration.inMilliseconds, RateLimitParser.httpRetryAfterDefaultDelay.inMilliseconds); }); + + test('do not parse namespaces if not metric_bucket', () { + final sut = + RateLimitParser('1:transaction:organization:quota_exceeded:custom') + .parseRateLimitHeader(); + + expect(sut.length, 1); + expect(sut[0].category, DataCategory.transaction); + expect(sut[0].namespaces, isEmpty); + }); + + test('parse namespaces on metric_bucket', () { + final sut = + RateLimitParser('1:metric_bucket:organization:quota_exceeded:custom') + .parseRateLimitHeader(); + + expect(sut.length, 1); + expect(sut[0].category, DataCategory.metricBucket); + expect(sut[0].namespaces, isNotEmpty); + expect(sut[0].namespaces.first, 'custom'); + }); + + test('parse empty namespaces on metric_bucket', () { + final sut = + RateLimitParser('1:metric_bucket:organization:quota_exceeded:') + .parseRateLimitHeader(); + + expect(sut.length, 1); + expect(sut[0].category, DataCategory.metricBucket); + expect(sut[0].namespaces, isEmpty); + }); + + test('parse missing namespaces on metric_bucket', () { + final sut = RateLimitParser('1:metric_bucket').parseRateLimitHeader(); + + expect(sut.length, 1); + expect(sut[0].category, DataCategory.metricBucket); + expect(sut[0].namespaces, isEmpty); + }); }); group('parseRetryAfterHeader', () { diff --git a/dart/test/protocol/rate_limiter_test.dart b/dart/test/protocol/rate_limiter_test.dart index cc931d5b66..1f52a60003 100644 --- a/dart/test/protocol/rate_limiter_test.dart +++ b/dart/test/protocol/rate_limiter_test.dart @@ -228,6 +228,118 @@ void main() { expect(fixture.mockRecorder.category, DataCategory.transaction); expect(fixture.mockRecorder.reason, DiscardReason.rateLimitBackoff); }); + + test('dropping of metrics recorded', () { + final rateLimiter = fixture.getSut(); + + final metricsItem = SentryEnvelopeItem.fromMetrics({}); + final eventEnvelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [metricsItem], + ); + + rateLimiter.updateRetryAfterLimits( + '1:metric_bucket:key, 5:metric_bucket:organization', null, 1); + + final result = rateLimiter.filter(eventEnvelope); + expect(result, isNull); + + expect(fixture.mockRecorder.category, DataCategory.metricBucket); + expect(fixture.mockRecorder.reason, DiscardReason.rateLimitBackoff); + }); + + group('apply rateLimit', () { + test('error', () { + final rateLimiter = fixture.getSut(); + fixture.dateTimeToReturn = 0; + + final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent()); + final envelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [eventItem], + ); + + rateLimiter.updateRetryAfterLimits( + '1:error:key, 5:error:organization', null, 1); + + expect(rateLimiter.filter(envelope), isNull); + }); + + test('transaction', () { + final rateLimiter = fixture.getSut(); + fixture.dateTimeToReturn = 0; + + final transaction = fixture.getTransaction(); + final eventItem = SentryEnvelopeItem.fromTransaction(transaction); + final envelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [eventItem], + ); + + rateLimiter.updateRetryAfterLimits( + '1:transaction:key, 5:transaction:organization', null, 1); + + final result = rateLimiter.filter(envelope); + expect(result, isNull); + }); + + test('metrics', () { + final rateLimiter = fixture.getSut(); + fixture.dateTimeToReturn = 0; + + final metricsItem = SentryEnvelopeItem.fromMetrics({}); + final envelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [metricsItem], + ); + + rateLimiter.updateRetryAfterLimits( + '1:metric_bucket:key, 5:metric_bucket:organization', null, 1); + + final result = rateLimiter.filter(envelope); + expect(result, isNull); + }); + + test('metrics with empty namespaces', () { + final rateLimiter = fixture.getSut(); + fixture.dateTimeToReturn = 0; + + final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent()); + final metricsItem = SentryEnvelopeItem.fromMetrics({}); + final envelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [eventItem, metricsItem], + ); + + rateLimiter.updateRetryAfterLimits( + '10:metric_bucket:key:quota_exceeded:', null, 1); + + final result = rateLimiter.filter(envelope); + expect(result, isNotNull); + expect(result!.items.length, 1); + expect(result.items.first.header.type, 'event'); + }); + + test('metrics with custom namespace', () { + final rateLimiter = fixture.getSut(); + fixture.dateTimeToReturn = 0; + + final eventItem = SentryEnvelopeItem.fromEvent(SentryEvent()); + final metricsItem = SentryEnvelopeItem.fromMetrics({}); + final envelope = SentryEnvelope( + SentryEnvelopeHeader.newEventId(), + [eventItem, metricsItem], + ); + + rateLimiter.updateRetryAfterLimits( + '10:metric_bucket:key:quota_exceeded:custom', null, 1); + + final result = rateLimiter.filter(envelope); + expect(result, isNotNull); + expect(result!.items.length, 1); + expect(result.items.first.header.type, 'event'); + }); + }); } class Fixture {