diff --git a/README.md b/README.md index 13b81883..efd74d93 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,27 @@ c.labels('get', '/') c.labels('post', '/submit') ``` +### Exemplars + +Exemplars can be added to counter and histogram metrics. Exemplars can be +specified by passing a dict of label value pairs to be exposed as the exemplar. +For example with a counter: + +```python +from prometheus_client import Counter +c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint']) +c.labels('get', '/').inc(exemplar={'trace_id': 'abc123'}) +c.labels('post', '/submit').inc(1.0, {'trace_id': 'def456'}) +``` + +And with a histogram: + +```python +from prometheus_client import Histogram +h = Histogram('request_latency_seconds', 'Description of histogram') +h.observe(4.7, {'trace_id': 'abc123'}) +``` + ### Process Collector The Python client automatically exports metrics about process CPU usage, RAM, @@ -510,6 +531,7 @@ This comes with a number of limitations: - Info and Enum metrics do not work - The pushgateway cannot be used - Gauges cannot use the `pid` label +- Exemplars are not supported There's several steps to getting this working: diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index 062fecff..399830ae 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -10,6 +10,7 @@ RESERVED_METRIC_LABEL_NAME_RE, ) from .registry import REGISTRY +from .samples import Exemplar from .utils import floatToGoString, INF if sys.version_info > (3,): @@ -36,18 +37,32 @@ def _build_full_name(metric_type, name, namespace, subsystem, unit): return full_name +def _validate_labelname(l): + if not METRIC_LABEL_NAME_RE.match(l): + raise ValueError('Invalid label metric name: ' + l) + if RESERVED_METRIC_LABEL_NAME_RE.match(l): + raise ValueError('Reserved label metric name: ' + l) + + def _validate_labelnames(cls, labelnames): labelnames = tuple(labelnames) for l in labelnames: - if not METRIC_LABEL_NAME_RE.match(l): - raise ValueError('Invalid label metric name: ' + l) - if RESERVED_METRIC_LABEL_NAME_RE.match(l): - raise ValueError('Reserved label metric name: ' + l) + _validate_labelname(l) if l in cls._reserved_labelnames: raise ValueError('Reserved label metric name: ' + l) return labelnames +def _validate_exemplar(exemplar): + runes = 0 + for k, v in exemplar.items(): + _validate_labelname(k) + runes += len(k) + runes += len(v) + if runes > 128: + raise ValueError('Exemplar labels have %d UTF-8 characters, exceeding the limit of 128') + + class MetricWrapperBase(object): _type = None _reserved_labelnames = () @@ -76,8 +91,8 @@ def describe(self): def collect(self): metric = self._get_metric() - for suffix, labels, value in self._samples(): - metric.add_sample(self._name + suffix, labels, value) + for suffix, labels, value, timestamp, exemplar in self._samples(): + metric.add_sample(self._name + suffix, labels, value, timestamp, exemplar) return [metric] def __str__(self): @@ -202,8 +217,8 @@ def _multi_samples(self): metrics = self._metrics.copy() for labels, metric in metrics.items(): series_labels = list(zip(self._labelnames, labels)) - for suffix, sample_labels, value in metric._samples(): - yield (suffix, dict(series_labels + list(sample_labels.items())), value) + for suffix, sample_labels, value, timestamp, exemplar in metric._samples(): + yield (suffix, dict(series_labels + list(sample_labels.items())), value, timestamp, exemplar) def _child_samples(self): # pragma: no cover raise NotImplementedError('_child_samples() must be implemented by %r' % self) @@ -256,12 +271,15 @@ def _metric_init(self): self._labelvalues) self._created = time.time() - def inc(self, amount=1): + def inc(self, amount=1, exemplar=None): """Increment counter by the given amount.""" self._raise_if_not_observable() if amount < 0: raise ValueError('Counters can only be incremented by non-negative amounts.') self._value.inc(amount) + if exemplar: + _validate_exemplar(exemplar) + self._value.set_exemplar(Exemplar(exemplar, amount, time.time())) def count_exceptions(self, exception=Exception): """Count exceptions in a block of code or function. @@ -275,8 +293,8 @@ def count_exceptions(self, exception=Exception): def _child_samples(self): return ( - ('_total', {}, self._value.get()), - ('_created', {}, self._created), + ('_total', {}, self._value.get(), None, self._value.get_exemplar()), + ('_created', {}, self._created, None, None), ) @@ -399,12 +417,12 @@ def set_function(self, f): self._raise_if_not_observable() def samples(self): - return (('', {}, float(f())),) + return (('', {}, float(f()), None, None),) self._child_samples = create_bound_method(samples, self) def _child_samples(self): - return (('', {}, self._value.get()),) + return (('', {}, self._value.get(), None, None),) class Summary(MetricWrapperBase): @@ -470,9 +488,10 @@ def time(self): def _child_samples(self): return ( - ('_count', {}, self._count.get()), - ('_sum', {}, self._sum.get()), - ('_created', {}, self._created)) + ('_count', {}, self._count.get(), None, None), + ('_sum', {}, self._sum.get(), None, None), + ('_created', {}, self._created, None, None), + ) class Histogram(MetricWrapperBase): @@ -564,7 +583,7 @@ def _metric_init(self): self._labelvalues + (floatToGoString(b),)) ) - def observe(self, amount): + def observe(self, amount, exemplar=None): """Observe the given amount. The amount is usually positive or zero. Negative values are @@ -579,6 +598,9 @@ def observe(self, amount): for i, bound in enumerate(self._upper_bounds): if amount <= bound: self._buckets[i].inc(1) + if exemplar: + _validate_exemplar(exemplar) + self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time())) break def time(self): @@ -593,11 +615,11 @@ def _child_samples(self): acc = 0 for i, bound in enumerate(self._upper_bounds): acc += self._buckets[i].get() - samples.append(('_bucket', {'le': floatToGoString(bound)}, acc)) - samples.append(('_count', {}, acc)) + samples.append(('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar())) + samples.append(('_count', {}, acc, None, None)) if self._upper_bounds[0] >= 0: - samples.append(('_sum', {}, self._sum.get())) - samples.append(('_created', {}, self._created)) + samples.append(('_sum', {}, self._sum.get(), None, None)) + samples.append(('_created', {}, self._created, None, None)) return tuple(samples) @@ -634,7 +656,7 @@ def info(self, val): def _child_samples(self): with self._lock: - return (('_info', self._value, 1.0,),) + return (('_info', self._value, 1.0, None, None),) class Enum(MetricWrapperBase): @@ -692,7 +714,7 @@ def state(self, state): def _child_samples(self): with self._lock: return [ - ('', {self._name: s}, 1 if i == self._value else 0,) + ('', {self._name: s}, 1 if i == self._value else 0, None, None) for i, s in enumerate(self._states) ] diff --git a/prometheus_client/values.py b/prometheus_client/values.py index bccb38e9..842837c5 100644 --- a/prometheus_client/values.py +++ b/prometheus_client/values.py @@ -14,6 +14,7 @@ class MutexValue(object): def __init__(self, typ, metric_name, name, labelnames, labelvalues, **kwargs): self._value = 0.0 + self._exemplar = None self._lock = Lock() def inc(self, amount): @@ -24,10 +25,18 @@ def set(self, value): with self._lock: self._value = value + def set_exemplar(self, exemplar): + with self._lock: + self._exemplar = exemplar + def get(self): with self._lock: return self._value + def get_exemplar(self): + with self._lock: + return self._exemplar + def MultiProcessValue(process_identifier=os.getpid): """Returns a MmapedValue class based on a process_identifier function. @@ -100,11 +109,19 @@ def set(self, value): self._value = value self._file.write_value(self._key, self._value) + def set_exemplar(self, exemplar): + # TODO: Implement exemplars for multiprocess mode. + return + def get(self): with lock: self.__check_for_pid_change() return self._value + def get_exemplar(self): + # TODO: Implement exemplars for multiprocess mode. + return None + return MmapedValue diff --git a/tests/openmetrics/test_exposition.py b/tests/openmetrics/test_exposition.py index 72578a0b..3b61e219 100644 --- a/tests/openmetrics/test_exposition.py +++ b/tests/openmetrics/test_exposition.py @@ -70,7 +70,6 @@ def test_summary(self): # EOF """, generate_latest(self.registry)) - @unittest.skipIf(sys.version_info < (2, 7), "Test requires Python 2.7+.") def test_histogram(self): s = Histogram('hh', 'A histogram', registry=self.registry) s.observe(0.05) @@ -114,37 +113,28 @@ def test_histogram_negative_buckets(self): """, generate_latest(self.registry)) def test_histogram_exemplar(self): - class MyCollector(object): - def collect(self): - metric = Metric("hh", "help", 'histogram') - # This is not sane, but it covers all the cases. - metric.add_sample("hh_bucket", {"le": "1"}, 0, None, Exemplar({'a': 'b'}, 0.5)) - metric.add_sample("hh_bucket", {"le": "2"}, 0, None, Exemplar({'le': '7'}, 0.5, 12)) - metric.add_sample("hh_bucket", {"le": "3"}, 0, 123, Exemplar({'a': 'b'}, 2.5, 12)) - metric.add_sample("hh_bucket", {"le": "4"}, 0, None, Exemplar({'a': '\n"\\'}, 3.5)) - metric.add_sample("hh_bucket", {"le": "+Inf"}, 0, None, None) - yield metric - - self.registry.register(MyCollector()) - self.assertEqual(b"""# HELP hh help + s = Histogram('hh', 'A histogram', buckets=[1, 2, 3, 4], registry=self.registry) + s.observe(0.5, {'a': 'b'}) + s.observe(1.5, {'le': '7'}) + s.observe(2.5, {'a': 'b'}) + s.observe(3.5, {'a': '\n"\\'}) + print(generate_latest(self.registry)) + self.assertEqual(b"""# HELP hh A histogram # TYPE hh histogram -hh_bucket{le="1"} 0.0 # {a="b"} 0.5 -hh_bucket{le="2"} 0.0 # {le="7"} 0.5 12 -hh_bucket{le="3"} 0.0 123 # {a="b"} 2.5 12 -hh_bucket{le="4"} 0.0 # {a="\\n\\"\\\\"} 3.5 -hh_bucket{le="+Inf"} 0.0 +hh_bucket{le="1.0"} 1.0 # {a="b"} 0.5 123.456 +hh_bucket{le="2.0"} 2.0 # {le="7"} 1.5 123.456 +hh_bucket{le="3.0"} 3.0 # {a="b"} 2.5 123.456 +hh_bucket{le="4.0"} 4.0 # {a="\\n\\"\\\\"} 3.5 123.456 +hh_bucket{le="+Inf"} 4.0 +hh_count 4.0 +hh_sum 8.0 +hh_created 123.456 # EOF """, generate_latest(self.registry)) def test_counter_exemplar(self): - class MyCollector(object): - def collect(self): - metric = Metric("cc", "A counter", 'counter') - metric.add_sample("cc_total", {}, 1, None, Exemplar({'a': 'b'}, 1.0, 123.456)) - metric.add_sample("cc_created", {}, 123.456, None, None) - yield metric - - self.registry.register(MyCollector()) + c = Counter('cc', 'A counter', registry=self.registry) + c.inc(exemplar={'a': 'b'}) self.assertEqual(b"""# HELP cc A counter # TYPE cc counter cc_total 1.0 # {a="b"} 1.0 123.456 diff --git a/tests/test_core.py b/tests/test_core.py index dbb27033..d8bed8a0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,3 +1,4 @@ +# coding=utf-8 from __future__ import unicode_literals from concurrent.futures import ThreadPoolExecutor @@ -45,7 +46,7 @@ def test_increment(self): self.assertEqual(1, self.registry.get_sample_value('c_total')) self.counter.inc(7) self.assertEqual(8, self.registry.get_sample_value('c_total')) - + def test_repr(self): self.assertEqual(repr(self.counter), "prometheus_client.metrics.Counter(c)") @@ -98,6 +99,28 @@ def test_inc_not_observable(self): counter = Counter('counter', 'help', labelnames=('label',), registry=self.registry) assert_not_observable(counter.inc) + def test_exemplar_invalid_label_name(self): + self.assertRaises(ValueError, self.counter.inc, exemplar={':o)': 'smile'}) + self.assertRaises(ValueError, self.counter.inc, exemplar={'1': 'number'}) + + def test_exemplar_unicode(self): + # 128 characters should not raise, even using characters larger than 1 byte. + self.counter.inc(exemplar={ + 'abcdefghijklmnopqrstuvwxyz': '26+16 characters', + 'x123456': '7+15 characters', + 'zyxwvutsrqponmlkjihgfedcba': '26+16 characters', + 'unicode': '7+15 chars 平', + }) + + def test_exemplar_too_long(self): + # 129 characters should fail. + self.assertRaises(ValueError, self.counter.inc, exemplar={ + 'abcdefghijklmnopqrstuvwxyz': '26+16 characters', + 'x1234567': '8+15 characters', + 'zyxwvutsrqponmlkjihgfedcba': '26+16 characters', + 'y123456': '7+15 characters', + }) + class TestGauge(unittest.TestCase): def setUp(self): @@ -415,6 +438,19 @@ def test_block_decorator(self): self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) + def test_exemplar_invalid_label_name(self): + self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={':o)': 'smile'}) + self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={'1': 'number'}) + + def test_exemplar_too_long(self): + # 129 characters in total should fail. + self.assertRaises(ValueError, self.histogram.observe, 1.0, exemplar={ + 'abcdefghijklmnopqrstuvwxyz': '26+16 characters', + 'x1234567': '8+15 characters', + 'zyxwvutsrqponmlkjihgfedcba': '26+16 characters', + 'y123456': '7+15 characters', + }) + class TestInfo(unittest.TestCase): def setUp(self):