From 61e1003c22242eb71570d89b8a7e56977b4c30d2 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 18 Jan 2020 07:45:44 -0500 Subject: [PATCH] removed ao backend, added additional tests, added generic backend class --- CHANGELOG.md | 8 + README.md | 84 +- requirements.txt | 3 +- setup.py | 4 +- timeseriesql/backends/ao_backend.py | 272 ----- timeseriesql/plot.py | 90 ++ timeseriesql/tests/ao_backend_test.py | 174 --- timeseriesql/tests/ast_test.py | 3 + timeseriesql/tests/csv_backend_test.py | 2 + timeseriesql/tests/json/ao_basic.json | 1462 ------------------------ timeseriesql/tests/query_test.py | 3 + timeseriesql/tests/timeseries_test.py | 5 +- timeseriesql/timeseries.py | 197 +--- timeseriesql/utils/__init__.py | 90 ++ 14 files changed, 263 insertions(+), 2134 deletions(-) delete mode 100644 timeseriesql/backends/ao_backend.py create mode 100644 timeseriesql/plot.py delete mode 100644 timeseriesql/tests/ao_backend_test.py delete mode 100644 timeseriesql/tests/json/ao_basic.json diff --git a/CHANGELOG.md b/CHANGELOG.md index f16f510..2095acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.5] - 2020-01-18 + +### Added +- Added generic class to build plotting libraries against the TimeSeries class + +### Removed +- removed the AppOptics backend into it's own repo + ## [0.1.4] - 2020-01-11 ### Added diff --git a/README.md b/README.md index 74b4caa..989121f 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ * [Installation](#installation) * [Usage](#usage) * [CSV Backend](#csv-backend-usage) - * [AppOptics Backend](#appoptics-backend-usage) * [TimeSeries](#timeseries-usage) +* [Plotting Libs](#plotting_libs) * [Roadmap](#roadmap) * [Contributing](#contributing) * [License](#license) @@ -125,7 +125,7 @@ If any columns are empty or don't contain a numeric value, the value becomes a ` #### Basic CSV Usage ```python -from timeseriesql.backends import CSVBackend +from timeseriesql.backends.csv_backend import CSVBackend data = CSVBackend(x for x in "path/to.csv")[:] ``` @@ -161,68 +161,10 @@ data = CSVBackend(x for x in "path/to.csv").labels( )[:] ``` -### AppOptics Backend Usage - -[Appoptics](www.appoptics.com) is a commercial time series database product. The backend converts a query into an -API call. - -The backend expects a ``APPOPTICS_TOKEN`` environment variable to be set in order to authenticate to AppOptics. - -#### AppOptics Query - -```python -from timeseriesql.backends.ao_backend import AOBackend - -data = AOBackend(x for x in "metric.name")['1h'] #basic -data = AOBackend(x * 100 for x in "metric.name")['1h'] #binary operations (+, -, /, *) -data = AOBackend(x * 1.8 + 32 for x in "metric.name")['1h'] #multiple binary operations (°C to °F) -data = AOBackend(x.max for x in "metric.name")[3600:] #get max value -``` - -#### AppOptics Filtering - -Currently only ``==`` and ``!=`` are supported. - -```python -from timeseriesql.backends.ao_backend import AOBackend - -data = AOBackend(x for x in "metric.name" if x.environment == 'production')[3600:] -``` - -#### AppOptics Grouping - -```python -from timeseriesql.backends.ao_backend import AOBackend - -data = AOBackend(x for x in "metric.name").group('environment')[3600:] -data = AOBackend(x - y for x,y in AOBackend((x.max for x in "metric1"), (x.min for x in "metric2")).by('tag1'))[3600:] -``` - -#### AppOptics Time -```python -from timeseriesql.backends.ao_backend import AOBackend - -data = AOBackend(x for x in "metric.name")[:] #no start or end time (not recommended) -data = AOBackend(x for x in "metric.name")[3600:] #from now - 3600 seconds until now, resolution of 1 -data = AOBackend(x for x in "metric.name")[3600:1800] #from now - 3600 seconds until now - 1800 seconds, resolution of 1 -data = AOBackend(x for x in "metric.name")[3600::300] #from now - 3600 seconds until now resoultion of 300 seconds -``` - -#### AppOptics Functions -```python -data = AOBackend(sum(derive(x)) for x in "metric.name")[3600:] #get the sums of the derivatives -data = AOBackend(zero_fill(x) for x in "metric.name")[3600::60] #zero_fill -``` - -#### AppOptics Raw Composite -```python -data = AOBackend('s("some_metric", "*")')[3600:] -``` - ### TimeSeries Usage The `TimeSeries` object is allows for manipulation of the time series data after the it's been queried from the -backend. There are also helper functions to convert to a pandas `DataFrame` and plot using matplotlib. +backend. In the following examples, the variables starting with `ts_` are assumed to be queried data from a backend. @@ -310,6 +252,26 @@ The conversion returns 2 pandas DataFrames, one for the labels and the other for data, labels = ts_1.to_pandas() ``` + +## Plotting Libs + +### Available + +* [matplotlib](https://github.com/mbeale/tiemseriesql-matplotlib) + +### Creating a custom backend + +Start by extending the [Plot](https://github.com/mbeale/timeseriesql/blob/master/timeseriesql/plot.py) class. + +```python +from timeseries.plot import Plot +class NewPlottingLib(Plot): + pass +``` + +There is a list of functions that can be extended for as different plots. Also there are functions that generate titles, xlabel, ylabel, and legend labels. Use those to grab default information. They can be overridden to provide more custom logic around the those fields. + + ## Roadmap diff --git a/requirements.txt b/requirements.txt index 18ce7ec..296d654 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -numpy -requests \ No newline at end of file +numpy \ No newline at end of file diff --git a/setup.py b/setup.py index 491a87a..91a8672 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="timeseriesql", - version="0.1.4", + version="0.1.5", description="A Pythonic query language for time series data", long_description=long_description, long_description_content_type="text/markdown", @@ -23,7 +23,7 @@ "Operating System :: OS Independent", "Development Status :: 3 - Alpha", ], - install_requires=["numpy", "requests"], + install_requires=["numpy"], python_requires=">=3.6", ) diff --git a/timeseriesql/backends/ao_backend.py b/timeseriesql/backends/ao_backend.py deleted file mode 100644 index 8effd39..0000000 --- a/timeseriesql/backends/ao_backend.py +++ /dev/null @@ -1,272 +0,0 @@ -import os -import requests -import numbers -import time -import re -from timeseriesql.query import Query -from timeseriesql.timeseries import TimeSeries -from timeseriesql.ast import Metric, Value - - -def create_scalar_time_series(series, scalar): - """ - use a trick to create a scalar time series by dividing a timeseries by itself and then - scaling it by the desired scalar - """ - return f'scale(divide([{series},{series}]),{{"factor":"{scalar}"}})' - - -def power_time_series(series, scalar): - """ Multiply a series by itself X times where X is a scalar """ - s = str(series) - return f"multiply([{','.join([s for _ in range(scalar)])}])" - -def modulo_time_series(series, scalar): - """ Get the modulo of series. num - divisor * floor(num / divisor)) """ - scalar_series = create_scalar_time_series(series,scalar) - return f"subtract([{series},multiply([{scalar_series},floor(divide([{series},{scalar_series}]))])])" - -def binary_operation(left, right, optype): - """ Handle binary operations """ - op_handlers = { - "BINARY_MULTIPLY": "multiply", - "BINARY_SUBTRACT": "subtract", - "BINARY_POWER": "power", - "BINARY_TRUE_DIVIDE": "divide", - "BINARY_FLOOR_DIVIDE": "divide", - "BINARY_MATRIX_MULTIPLY": "multiply", - "BINARY_MODULO": "modulo", - "BINARY_ADD": "sum", - } - - is_right_scalar = isinstance(right, numbers.Number) - is_left_scalar = isinstance(left, numbers.Number) - - opcode = op_handlers.get(optype, None) - if opcode: - if opcode in ["power", "modulo"]: - op_func = power_time_series - if opcode == 'modulo': - op_func = modulo_time_series - if is_left_scalar: - return op_func(right, left) - elif is_right_scalar: - return op_func(left, right) - else: - return ValueError(f"There was not a scalar present for the {opcode} operation") - else: - if is_left_scalar: - return f"{opcode}([{create_scalar_time_series(right, left)}, {right}])" - elif is_right_scalar: - return f"{opcode}([{left},{create_scalar_time_series(left, right)}])" - else: - return f"{opcode}([{left},{right}])" - raise TypeError(f"{optype} is not a supported operation") - - -class CompositeDefinition: - """A class for composite definitions""" - - def __init__(self, name, resolution=1): - self.name = name - self.filter = [] - self.resolution = resolution - self.sum_func = "mean" - - def __str__(self): - filters = '"*"' - if len(self.filter) > 0: - filters = "{" - quote = '"' - for x in self.filter: - val = x["value"] if x["op"] == "==" else "!" + x["value"] - filters += f"{quote}{x['name']}{quote}:{quote}{val}{quote}" - filters += "}" - return f's("{self.name}",{filters},{{period:"{self.resolution}","function":"{self.sum_func}"}})' - - -class AOBackend(Query): - """ Extends the Query object to translate execution plans into the AppOptics Composite language """ - - BASE_URL = "https://api.appoptics.com/v1/" - API_TOKEN = os.environ.get("APPOPTICS_TOKEN", "") - COMPOSITE_DEF = None - - def __init__(self, *args): - if len(args) == 1 and isinstance(args[0], str): - self.COMPOSITE_DEF = args[0] - super(AOBackend, self).__init__(*args) - - @classmethod - def get(cls, endpoint, params): - """issue a get request""" - query_str = "?" + "&".join(["{}={}".format(key, value) for key, value in params.items()]) - response = requests.get( - cls.BASE_URL + endpoint + query_str, auth=requests.auth.HTTPBasicAuth(cls.API_TOKEN, "") - ) - if response.status_code > 399: - raise ConnectionError("{} - {}".format(response.status_code, response.text)) - return response - - @classmethod - def post(cls, endpoint, body): - """issue a post request""" - response = requests.post( - cls.BASE_URL + endpoint, json=body, auth=requests.auth.HTTPBasicAuth(cls.API_TOKEN, "") - ) - if response.status_code > 399: - raise ValueError("{} - {}".format(response.status_code, response.text)) - return response - - @property - def composite(self): - if not self.COMPOSITE_DEF: - now = int(time.time()) - plan = self._generate_plan() - self.COMPOSITE_DEF = self.create_query( - plan, {"start_time": now - 3600, "end_time": now, "resolution": 1} - ) - return self.COMPOSITE_DEF - - # AST Class handling - - def binaryadd(self, left, right, period): - return binary_operation(left, right, "BINARY_ADD") - - def binaryfloordivide(self, left, right, period): - return binary_operation(left, right, "BINARY_FLOOR_DIVIDE") - - def binarymatrixmultiply(self, left, right, period): - return binary_operation(left, right, "BINARY_MATRIX_MULTIPLY") - - def binarymodulo(self, left, right, period): - return binary_operation(left, right, "BINARY_MODULO") - - def binarymultiply(self, left, right, period): - return binary_operation(left, right, "BINARY_MULTIPLY") - - def binarypower(self, left, right, period): - return binary_operation(left, right, "BINARY_POWER") - - def binarysubtract(self, left, right, period): - return binary_operation(left, right, "BINARY_SUBTRACT") - - def binarytruedivide(self, left, right, period): - return binary_operation(left, right, "BINARY_TRUE_DIVIDE") - - def compareequal(self, left, right, period): - return {"name": left, "value": right, "op": "=="} - - def comparenotequal(self, left, right, period): - return {"name": left, "value": right, "op": "!="} - - def filter(self, left, right, period): - left.filter.append(right) - return left - - def funcargs(self, left, right, period): - kwargs = "" - if len(right) > 0: - kwargs = ",{" - for k, v in right.items(): - if kwargs != ",{": - kwargs += "," - kwargs += '"' + k + '":"' + str(v) + '"' - kwargs += "}" - left = f"{self.traverse_tree(left[0], period)}" - return f"{left}{kwargs}" - - def funccall(self, left, right, period): - valid_func_names = [ - "abs", - "bottom", - "ceiling", - "derive", - "fill", - "filter", - "floor", - "integrate", - "last_fill", - "max", - "mean", - "min", - "moving_average", - "rate", - "sum", - "top", - "window", - "zero_fill", - ] - if callable(left): - left = left.__name__ - if left not in valid_func_names: - raise NotImplementedError(f"{left} is not a valid function") - return f"{left}({right})" - - def group(self, left, right, period): - if len(right[0]) == 0: - raise AttributeError("Must have at least one label to group by") - labels = '"' + ",".join(right[0]) + '"' - if isinstance(left, str): - sets = [] - for m in re.finditer("s\(([^)]+)\)", left): - sets.append((m.start(), m.end())) - for s in reversed(sets): - left = left[0 : s[0]] + f"mean({left[s[0]:s[1]]})" + left[s[1] :] - return f"group_by({labels},{left})" - - def loadattr(self, left, right, period): - if isinstance(left, CompositeDefinition): - left.sum_func = right - return left - raise NotImplementedError - - # end AST class handling - - def traverse_tree(self, root, period): - if root: - if isinstance(root, Metric): - return CompositeDefinition(root.value, resolution=period["resolution"]) - if isinstance(root, Value): - return root.value - else: - left = self.traverse_tree(root.left, period) - right = self.traverse_tree(root.right, period) - try: - op = self.__getattribute__(root.__class__.__name__.lower()) - return op(left, right, period) - except Exception as e: - raise NotImplementedError( - f"AST class of {root.__class__.__name__} is not supported" - ) - - def create_query(self, plan, period): - """Create a composite based on the plan and period""" - - query = "" - query = self.traverse_tree(plan, period) - return str(query) - - def execute_plan(self): - """Execute the plan and return a TimeSeries object for any further processing or displaying""" - params = self._process_period() - if self.COMPOSITE_DEF: - params["compose"] = self.COMPOSITE_DEF - else: - plan = self._generate_plan() - self.COMPOSITE_DEF = self.create_query(plan, params) - params["compose"] = self.COMPOSITE_DEF - series = self.get("measurements", params) - timeseries = None - for stream in series.json()["series"]: - labels = {**stream["tags"]} - if "metric" in stream: - labels["name"] = stream["metric"]["name"] - time, data = zip(*[(m["time"], m["value"]) for m in stream["measurements"]]) - t = TimeSeries(shape=(len(stream["measurements"]), 1), labels=labels, time=time) - t[:, 0] = data - if timeseries is not None: - timeseries = timeseries.merge([t]) - else: - timeseries = t.copy() - return timeseries diff --git a/timeseriesql/plot.py b/timeseriesql/plot.py new file mode 100644 index 0000000..494752b --- /dev/null +++ b/timeseriesql/plot.py @@ -0,0 +1,90 @@ +from dataclasses import dataclass +from typing import Callable, Any, Dict +from .timeseries import TimeSeries + + +def generate_title(ts: TimeSeries) -> str: + """Generate a title based on common tags""" + # get common labels + labels = ts._get_unique_keys() + if len(labels) > 0: + return ".".join([str(v) for k, v in labels.items()]) + return "TimeSeries" + + +def generate_xlabel(ts: TimeSeries) -> str: + return "Value" + + +def generate_ylabel(ts: TimeSeries) -> str: + return "Value" + + +def generate_legend_labels(ts: TimeSeries) -> Dict[str, str]: + common_labels = ts._get_unique_keys().keys() + labels = [] + for l in ts.labels: + labels.append(".".join([str(v) for k, v in l.items() if k not in common_labels])) + return labels + + +def generate_chart_options(ts: TimeSeries) -> Dict[str, str]: + return {} + + +@dataclass +class Plot: + + title_func: Callable[[Any], str] = generate_title + xlabel_func: Callable[[Any], str] = generate_xlabel + ylabel_func: Callable[[Any], str] = generate_ylabel + legend_labels_func: Callable[[Any], str] = generate_legend_labels + chart_options_func: Callable[[Any], str] = generate_chart_options + + def line_plot(self, ts: TimeSeries, **kwargs): + """ Plot all TimeSeries columns as a line """ + return NotImplementedError + + def dist_plot(self, ts: TimeSeries, **kwargs): + """ Plot all TimeSeries columns as a distribution """ + return NotImplementedError + + def stacked_plot(self, ts: TimeSeries, **kwargs): + """ Plot all TimeSeries columns as a stacked chart """ + return NotImplementedError + + def timebox_plot(self, ts: TimeSeries, **kwargs): + """ Plot all TimeSeries columns as a time box chart """ + return NotImplementedError + + def correlogram_plot(self, ts: TimeSeries, **kwargs): + """ Plot all auto-correlation chart """ + return NotImplementedError + + def text_plot(self, ts: TimeSeries, **kwargs): + """ Plot a single value """ + return NotImplementedError + + def heatmap_plot(self, ts: TimeSeries, **kwargs): + """ Plot all TimeSeries columns as a heatmap """ + raise NotImplementedError + + def pacf(self, ts: TimeSeries, **kwargs): + """ Plot a pacf chart """ + raise NotImplementedError + + def acf(self, ts: TimeSeries, **kwargs): + """ Plot a acf chart """ + raise NotImplementedError + + def qq_plot(self, ts: TimeSeries, **kwargs): + """ Plot a qq plot """ + raise NotImplementedError + + def probability_plot(self, ts: TimeSeries, **kwargs): + """ Plot a probablility plot """ + raise NotImplementedError + + def polar_plot(self, ts: TimeSeries, **kwargs): + """ Plot a polar plot """ + raise NotImplementedError diff --git a/timeseriesql/tests/ao_backend_test.py b/timeseriesql/tests/ao_backend_test.py deleted file mode 100644 index 6974482..0000000 --- a/timeseriesql/tests/ao_backend_test.py +++ /dev/null @@ -1,174 +0,0 @@ -import unittest -import time -import json -from unittest import mock -from timeseriesql.backends.ao_backend import AOBackend, create_scalar_time_series - - -def mocked_requests_get(*args, **kwargs): - class MockResponse: - def __init__(self, json_data, status_code, response_text=None): - self.json_data = json_data - self.status_code = status_code - self.text = response_text - - def json(self): - with open("./timeseriesql/tests/json/ao_basic.json") as json_file: - data = json.load(json_file) - - return data - - return MockResponse("ao_basic.json", 200) - - -class TestAOBackend(unittest.TestCase): - def setUp(self): - self.maxDiff = None - now = int(time.time()) - self.basic_period = {"start_time": now - 3600, "end_time": now, "resolution": 1} - - def test_execute_basic_plan(self): - a = AOBackend(x for x in "test") - self.assertEqual(a.composite, 's("test","*",{period:"1","function":"mean"})') - - def test_basic_query_with_label(self): - - a = AOBackend(x.max for x in "test") - self.assertEqual(a.composite, 's("test","*",{period:"1","function":"max"})') - - def test_basic_filter(self): - a = AOBackend(x for x in "test" if x.label1 == "prod") - self.assertEqual(a.composite, 's("test",{"label1":"prod"},{period:"1","function":"mean"})') - - a = AOBackend(x for x in "test" if x.label1 != "prod") - self.assertEqual(a.composite, 's("test",{"label1":"!prod"},{period:"1","function":"mean"})') - - a = AOBackend(x for x in "test" if x.label1 in ["prod", "prod2"]) - self.assertRaises(NotImplementedError, getattr, a, "composite") - - a = AOBackend(x for x in "test" if "prod" == "prod") - self.assertRaises(NotImplementedError, getattr, a, "composite") - - def test_binary_operations(self): - a = AOBackend(x * 100 for x in "test" if x.label1 == "prod") - self.assertEqual( - a.composite, - 'multiply([s("test",{"label1":"prod"},{period:"1","function":"mean"}),scale(divide([s("test",{"label1":"prod"},{period:"1","function":"mean"}),s("test",{"label1":"prod"},{period:"1","function":"mean"})]),{"factor":"100"})])', - ) - - def test_power_operation(self): - a = AOBackend(x ** 3 for x in "test" if x.label1 == "prod") - self.assertEqual( - a.composite, - 'multiply([s("test",{"label1":"prod"},{period:"1","function":"mean"}),s("test",{"label1":"prod"},{period:"1","function":"mean"}),s("test",{"label1":"prod"},{period:"1","function":"mean"})])', - ) - - def test_modulo_operation(self): - a = AOBackend(x % 3 for x in "test" if x.label1 == "prod") - series = 's("test",{"label1":"prod"},{period:"1","function":"mean"})' - divisor = f'scale(divide([{series},{series}]),{{"factor":"3"}})' - self.assertEqual( - a.composite, - f"subtract([{series},multiply([{divisor},floor(divide([{series},{divisor}]))])])", - ) - - def test_group_by(self): - expected_value = 'group_by("label1",multiply([mean(s("test",{"label1":"prod"},{period:"1","function":"mean"})),scale(divide([mean(s("test",{"label1":"prod"},{period:"1","function":"mean"})),mean(s("test",{"label1":"prod"},{period:"1","function":"mean"}))]),{"factor":"100"})]))' - - a = AOBackend(x * 100 for x in "test" if x.label1 == "prod").by(["label1"]) - self.assertEqual(a.composite, expected_value) - - expected_value = 'group_by("label1,label2,label3",multiply([mean(s("test",{"label1":"prod"},{period:"1","function":"mean"})),scale(divide([mean(s("test",{"label1":"prod"},{period:"1","function":"mean"})),mean(s("test",{"label1":"prod"},{period:"1","function":"mean"}))]),{"factor":"100"})]))' - - a = AOBackend(x * 100 for x in "test" if x.label1 == "prod").by( - ["label1", "label2", "label3"] - ) - self.assertEqual(a.composite, expected_value) - - def test_composite_func(self): - - a = AOBackend(mean(x) for x in "test") - self.assertEqual(a.composite, 'mean(s("test","*",{period:"1","function":"mean"}))') - - a = AOBackend(bottom(x, function="min", count=10) for x in "test") - self.assertEqual( - a.composite, - 'bottom(s("test","*",{period:"1","function":"mean"}),{"count":"10","function":"min"})', - ) - - a = AOBackend(does_not_exist(x) for x in "test") - self.assertRaises(NotImplementedError, getattr, a, "composite") - - def test_multiple_binary_operations(self): - expected_value = 'multiply([multiply([s("test",{"label1":"prod"},{period:"1","function":"mean"}),scale(divide([s("test",{"label1":"prod"},{period:"1","function":"mean"}),s("test",{"label1":"prod"},{period:"1","function":"mean"})]),{"factor":"100"})]),s("test",{"label1":"prod"},{period:"1","function":"mean"})])' - - a = AOBackend(x * 100 * x for x in "test" if x.label1 == "prod") - self.assertEqual(a.composite, expected_value) - - expected_value2 = 'multiply([scale(divide([s("test",{"label1":"prod"},{period:"1","function":"mean"}),s("test",{"label1":"prod"},{period:"1","function":"mean"})]),{"factor":"100"}), s("test",{"label1":"prod"},{period:"1","function":"mean"})])' - a = AOBackend(100 * x for x in "test" if x.label1 == "prod") - self.assertEqual(a.composite, expected_value2) - - a = AOBackend(x ^ 3 for x in "test") - self.assertRaises(NotImplementedError, getattr, a, "composite") - - def test_scaler_create_trick(self): - self.maxDiff = None - series = 's("AWS.EC2.CPUUtilization",{"hostname":"ip-x.ec2.internal"},{period:"60","function":"mean"})' - expected_result = 'scale(divide([s("AWS.EC2.CPUUtilization",{"hostname":"ip-x.ec2.internal"},{period:"60","function":"mean"}),s("AWS.EC2.CPUUtilization",{"hostname":"ip-x.ec2.internal"},{period:"60","function":"mean"})]),{"factor":"100"})' - - actual_result = create_scalar_time_series(series, 100) - self.assertEqual(actual_result, expected_result) - - def test_sum_of_sums(self): - expected_value = 'sum(s("metric1","*",{period:"1","function":"sum"}))' - a = AOBackend(sum(x.sum) for x in "metric1") - self.assertEqual(a.composite, expected_value) - - def test_multiple_generators(self): - self.maxDiff = None - expected_value = 'group_by("tag1",subtract([mean(s("metric1","*",{period:"1","function":"max"})),mean(s("metric2","*",{period:"1","function":"min"}))]))' - a = AOBackend( - x - y for x, y in AOBackend((x.max for x in "metric1"), (x.min for x in "metric2")) - ).by("tag1") - self.assertEqual(a.composite, expected_value) - - def test_multiple_functions(self): - expected_value = 'sum(derive(s("test","*",{period:"1","function":"mean"})))' - a = AOBackend(sum(derive(x)) for x in "test") - self.assertEqual(a.composite, expected_value) - - def test_complex_composite_1(self): - import dis - - expected_result = 'multiply([divide([zero_fill(sum(s("metric1","*",{period:"1","function":"sum"}))),zero_fill(sum(s("metric2","*",{period:"1","function":"sum"})))]),scale(divide([divide([zero_fill(sum(s("metric1","*",{period:"1","function":"sum"}))),zero_fill(sum(s("metric2","*",{period:"1","function":"sum"})))]),divide([zero_fill(sum(s("metric1","*",{period:"1","function":"sum"}))),zero_fill(sum(s("metric2","*",{period:"1","function":"sum"})))])]),{"factor":"100"})])' - gen = ( - zero_fill(sum(x)) / zero_fill(sum(y)) * 100 - for x, y in AOBackend((x.sum for x in "metric1"), (x.sum for x in "metric2")) - ) - a = AOBackend(gen) - self.assertEqual(a.composite, expected_result) - - @mock.patch("timeseriesql.backends.ao_backend.requests.get", side_effect=mocked_requests_get) - def test_end_to_end_with_mock(self, mock_requests): - data = AOBackend(x for x in "test")[:] - - # test the labels - self.assertEqual(data.labels, [{"service": "service1", "name": "metric1"}]) - - # measurements loaded - self.assertEqual(len(data), 358) - self.assertEqual(data[-1][0], 549.56976923) - - @mock.patch("timeseriesql.backends.ao_backend.requests.get", side_effect=mocked_requests_get) - def test_raw_composite(self, mock_requests): - composite = 's("some_metric", "*")' - - a = AOBackend(composite) - self.assertEqual(a.composite, composite) - - data = AOBackend(composite)[:] - # measurements loaded - self.assertEqual(len(data), 358) - self.assertEqual(data[-1][0], 549.56976923) - diff --git a/timeseriesql/tests/ast_test.py b/timeseriesql/tests/ast_test.py index 8f4801a..d2a7761 100644 --- a/timeseriesql/tests/ast_test.py +++ b/timeseriesql/tests/ast_test.py @@ -216,3 +216,6 @@ def test_query_for_iterator(self): actual = a.decompile() expected = Metric('test') self.assertEqual(actual, expected) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/timeseriesql/tests/csv_backend_test.py b/timeseriesql/tests/csv_backend_test.py index 26f6207..29570f6 100644 --- a/timeseriesql/tests/csv_backend_test.py +++ b/timeseriesql/tests/csv_backend_test.py @@ -141,3 +141,5 @@ def test_load_csv_string_date(self): tstamps = [1571296500.0 + (i * 60) for i in range(11)] self.assertTrue(np.array_equal(data.time, tstamps)) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/timeseriesql/tests/json/ao_basic.json b/timeseriesql/tests/json/ao_basic.json deleted file mode 100644 index bec8503..0000000 --- a/timeseriesql/tests/json/ao_basic.json +++ /dev/null @@ -1,1462 +0,0 @@ -{ - "series": [ - { - "tags": { - "service": "service1" - }, - "measurements": [ - { - "time": 1570503180, - "value": 318.22474129353236 - }, - { - "time": 1570503240, - "value": 304.8096943722944 - }, - { - "time": 1570503300, - "value": 262.1919918699187 - }, - { - "time": 1570503360, - "value": 263.6059714285714 - }, - { - "time": 1570503420, - "value": 291.14184084084087 - }, - { - "time": 1570503480, - "value": 244.65009090909086 - }, - { - "time": 1570503540, - "value": 232.82217516629714 - }, - { - "time": 1570503600, - "value": 378.9781111111111 - }, - { - "time": 1570503660, - "value": 277.0845428571428 - }, - { - "time": 1570503720, - "value": 316.3781873963516 - }, - { - "time": 1570503780, - "value": 377.7483925299507 - }, - { - "time": 1570503840, - "value": 251.00682806573957 - }, - { - "time": 1570503900, - "value": 315.3370494440811 - }, - { - "time": 1570503960, - "value": 315.4636666666667 - }, - { - "time": 1570504020, - "value": 276.692440285205 - }, - { - "time": 1570504080, - "value": 280.125 - }, - { - "time": 1570504140, - "value": 280.3493466666667 - }, - { - "time": 1570504200, - "value": 346.78275824175824 - }, - { - "time": 1570504260, - "value": 426.46821212121216 - }, - { - "time": 1570504320, - "value": 300.41500784313723 - }, - { - "time": 1570504380, - "value": 262.9540237762237 - }, - { - "time": 1570504440, - "value": 247.38855492957745 - }, - { - "time": 1570504500, - "value": 245.2328875653083 - }, - { - "time": 1570504560, - "value": 274.19314814814817 - }, - { - "time": 1570504620, - "value": 238.97471403353927 - }, - { - "time": 1570504680, - "value": 321.7498370860927 - }, - { - "time": 1570504740, - "value": 316.4091827160494 - }, - { - "time": 1570504800, - "value": 242.0977368421053 - }, - { - "time": 1570504860, - "value": 394.31937441077446 - }, - { - "time": 1570504920, - "value": 343.2076545454546 - }, - { - "time": 1570504980, - "value": 298.0468249754179 - }, - { - "time": 1570505040, - "value": 240.93947117794485 - }, - { - "time": 1570505100, - "value": 405.86379999999997 - }, - { - "time": 1570505160, - "value": 372.9639885057472 - }, - { - "time": 1570505220, - "value": 322.87346476190476 - }, - { - "time": 1570505280, - "value": 289.18189523809525 - }, - { - "time": 1570505340, - "value": 273.5253111111111 - }, - { - "time": 1570505400, - "value": 384.214852442672 - }, - { - "time": 1570505460, - "value": 265.8288052691867 - }, - { - "time": 1570505520, - "value": 315.2489428571428 - }, - { - "time": 1570505580, - "value": 393.58811632453563 - }, - { - "time": 1570505640, - "value": 312.5399627791563 - }, - { - "time": 1570505700, - "value": 286.14058911564626 - }, - { - "time": 1570505760, - "value": 207.43975342465757 - }, - { - "time": 1570505820, - "value": 342.67930545454544 - }, - { - "time": 1570505880, - "value": 296.6422589147287 - }, - { - "time": 1570505940, - "value": 313.67002574257424 - }, - { - "time": 1570506000, - "value": 207.31994564102564 - }, - { - "time": 1570506060, - "value": 303.7188857142857 - }, - { - "time": 1570506120, - "value": 358.73194784580494 - }, - { - "time": 1570506180, - "value": 298.3379777777778 - }, - { - "time": 1570506240, - "value": 259.9978998946259 - }, - { - "time": 1570506300, - "value": 362.7998365667852 - }, - { - "time": 1570506360, - "value": 247.2177047619048 - }, - { - "time": 1570506420, - "value": 281.16301904761906 - }, - { - "time": 1570506480, - "value": 294.6742398190045 - }, - { - "time": 1570506540, - "value": 343.4889797979798 - }, - { - "time": 1570506600, - "value": 199.18817322834647 - }, - { - "time": 1570506660, - "value": 326.3781527084315 - }, - { - "time": 1570506720, - "value": 282.13413016560906 - }, - { - "time": 1570506780, - "value": 297.0490701754386 - }, - { - "time": 1570506840, - "value": 318.250091222031 - }, - { - "time": 1570506900, - "value": 217.9212101210121 - }, - { - "time": 1570506960, - "value": 274.0620060606061 - }, - { - "time": 1570507020, - "value": 248.40979417122043 - }, - { - "time": 1570507080, - "value": 321.3246435541859 - }, - { - "time": 1570507140, - "value": 209.08689128397376 - }, - { - "time": 1570507200, - "value": 249.13835163398696 - }, - { - "time": 1570507260, - "value": 220.9112153846154 - }, - { - "time": 1570507320, - "value": 299.6111547116737 - }, - { - "time": 1570507380, - "value": 313.1046085106383 - }, - { - "time": 1570507440, - "value": 257.31361573033706 - }, - { - "time": 1570507500, - "value": 356.2603112781955 - }, - { - "time": 1570507560, - "value": 250.94917518248172 - }, - { - "time": 1570507620, - "value": 305.04397440423656 - }, - { - "time": 1570507680, - "value": 273.1301440860215 - }, - { - "time": 1570507740, - "value": 264.26284149184147 - }, - { - "time": 1570507800, - "value": 293.70760660660665 - }, - { - "time": 1570507860, - "value": 253.69729841269842 - }, - { - "time": 1570507920, - "value": 319.191526401631 - }, - { - "time": 1570507980, - "value": 343.3214646464647 - }, - { - "time": 1570508040, - "value": 241.2081177748805 - }, - { - "time": 1570508100, - "value": 250.72975324675326 - }, - { - "time": 1570508160, - "value": 330.9445081967213 - }, - { - "time": 1570508220, - "value": 282.5760701298702 - }, - { - "time": 1570508280, - "value": 271.57534910941473 - }, - { - "time": 1570508340, - "value": 231.61444742268043 - }, - { - "time": 1570508400, - "value": 305.2279001389103 - }, - { - "time": 1570508460, - "value": 268.51434897360707 - }, - { - "time": 1570508520, - "value": 281.671928320802 - }, - { - "time": 1570508580, - "value": 366.3422121212121 - }, - { - "time": 1570508640, - "value": 272.5252857142857 - }, - { - "time": 1570508700, - "value": 323.6107554140127 - }, - { - "time": 1570508760, - "value": 160.75053505933118 - }, - { - "time": 1570508820, - "value": 277.62737710437716 - }, - { - "time": 1570508880, - "value": 309.1280023688663 - }, - { - "time": 1570508940, - "value": 305.83734601226996 - }, - { - "time": 1570509000, - "value": 345.5915922920892 - }, - { - "time": 1570509060, - "value": 298.0080222222222 - }, - { - "time": 1570509120, - "value": 256.483015037594 - }, - { - "time": 1570509180, - "value": 198.8200613756614 - }, - { - "time": 1570509240, - "value": 288.0532857142857 - }, - { - "time": 1570509300, - "value": 324.7212383838384 - }, - { - "time": 1570509360, - "value": 373.9867176470588 - }, - { - "time": 1570509420, - "value": 297.28237516005123 - }, - { - "time": 1570509480, - "value": 310.64291895424833 - }, - { - "time": 1570509540, - "value": 272.405452532833 - }, - { - "time": 1570509600, - "value": 306.241395101416 - }, - { - "time": 1570509660, - "value": 261.8115641025641 - }, - { - "time": 1570509720, - "value": 278.00866901172526 - }, - { - "time": 1570509780, - "value": 240.9990202020202 - }, - { - "time": 1570509840, - "value": 241.05458832565287 - }, - { - "time": 1570509900, - "value": 274.99444654088046 - }, - { - "time": 1570509960, - "value": 191.41309774436095 - }, - { - "time": 1570510020, - "value": 277.58558512396695 - }, - { - "time": 1570510080, - "value": 324.3263538461539 - }, - { - "time": 1570510140, - "value": 366.6136469135803 - }, - { - "time": 1570510200, - "value": 315.2094699961285 - }, - { - "time": 1570510260, - "value": 276.8763445378152 - }, - { - "time": 1570510320, - "value": 363.24872121212127 - }, - { - "time": 1570510380, - "value": 336.7454410058027 - }, - { - "time": 1570510440, - "value": 297.052425997426 - }, - { - "time": 1570510500, - "value": 247.9864293706294 - }, - { - "time": 1570510560, - "value": 271.3230368098159 - }, - { - "time": 1570510620, - "value": 297.88035042735044 - }, - { - "time": 1570510680, - "value": 284.8253139653415 - }, - { - "time": 1570510740, - "value": 278.08913519813524 - }, - { - "time": 1570510800, - "value": 290.3645690821256 - }, - { - "time": 1570510860, - "value": 239.96876278118611 - }, - { - "time": 1570510920, - "value": 226.9394672754947 - }, - { - "time": 1570510980, - "value": 239.67221758241763 - }, - { - "time": 1570511040, - "value": 345.1143903549816 - }, - { - "time": 1570511100, - "value": 307.3044603174604 - }, - { - "time": 1570511160, - "value": 273.476413711584 - }, - { - "time": 1570511220, - "value": 245.40474545454546 - }, - { - "time": 1570511280, - "value": 295.8398416988417 - }, - { - "time": 1570511340, - "value": 404.2960091503269 - }, - { - "time": 1570511400, - "value": 371.58003434343436 - }, - { - "time": 1570511460, - "value": 284.82677215189875 - }, - { - "time": 1570511520, - "value": 274.9723469387755 - }, - { - "time": 1570511580, - "value": 234.94124705882356 - }, - { - "time": 1570511640, - "value": 290.34854599015733 - }, - { - "time": 1570511700, - "value": 386.315561161915 - }, - { - "time": 1570511760, - "value": 235.26659370078744 - }, - { - "time": 1570511820, - "value": 226.56996456086284 - }, - { - "time": 1570511880, - "value": 333.23807407407406 - }, - { - "time": 1570511940, - "value": 315.56753731343287 - }, - { - "time": 1570512000, - "value": 327.4924117647059 - }, - { - "time": 1570512060, - "value": 320.1887671641791 - }, - { - "time": 1570512120, - "value": 292.6320855810856 - }, - { - "time": 1570512180, - "value": 254.8682046783626 - }, - { - "time": 1570512240, - "value": 260.24452647352643 - }, - { - "time": 1570512300, - "value": 347.7546235294118 - }, - { - "time": 1570512360, - "value": 349.52112293144205 - }, - { - "time": 1570512420, - "value": 341.18872890899945 - }, - { - "time": 1570512480, - "value": 228.67064247517192 - }, - { - "time": 1570512540, - "value": 254.93185154061624 - }, - { - "time": 1570512600, - "value": 260.0611909353905 - }, - { - "time": 1570512660, - "value": 275.9872680412371 - }, - { - "time": 1570512720, - "value": 279.1410455696202 - }, - { - "time": 1570512780, - "value": 264.2627915742794 - }, - { - "time": 1570512840, - "value": 223.75407692307692 - }, - { - "time": 1570512900, - "value": 241.55248571428572 - }, - { - "time": 1570512960, - "value": 221.18408433734945 - }, - { - "time": 1570513020, - "value": 280.98687240829344 - }, - { - "time": 1570513080, - "value": 200.02908045977009 - }, - { - "time": 1570513140, - "value": 280.9056907555721 - }, - { - "time": 1570513200, - "value": 306.1111483862328 - }, - { - "time": 1570513260, - "value": 356.0630842911877 - }, - { - "time": 1570513320, - "value": 243.94874306569344 - }, - { - "time": 1570513380, - "value": 279.23773294092 - }, - { - "time": 1570513440, - "value": 288.7153490909091 - }, - { - "time": 1570513500, - "value": 335.1241483253589 - }, - { - "time": 1570513560, - "value": 317.9943535353536 - }, - { - "time": 1570513620, - "value": 215.88774641148328 - }, - { - "time": 1570513680, - "value": 243.709 - }, - { - "time": 1570513740, - "value": 209.515 - }, - { - "time": 1570513800, - "value": 322.4291635220126 - }, - { - "time": 1570513860, - "value": 220.78647186147185 - }, - { - "time": 1570513920, - "value": 286.63328686868687 - }, - { - "time": 1570513980, - "value": 364.2124666666666 - }, - { - "time": 1570514040, - "value": 344.1403431013431 - }, - { - "time": 1570514100, - "value": 235.72267599346762 - }, - { - "time": 1570514160, - "value": 283.3941965811966 - }, - { - "time": 1570514220, - "value": 275.6097790849673 - }, - { - "time": 1570514280, - "value": 273.06385245901635 - }, - { - "time": 1570514340, - "value": 345.50537395517875 - }, - { - "time": 1570514400, - "value": 343.67529247311825 - }, - { - "time": 1570514460, - "value": 187.35780722891562 - }, - { - "time": 1570514520, - "value": 300.4283992337165 - }, - { - "time": 1570514580, - "value": 260.04506162464986 - }, - { - "time": 1570514640, - "value": 247.1252273249139 - }, - { - "time": 1570514700, - "value": 354.65442222222225 - }, - { - "time": 1570514760, - "value": 304.24026868686866 - }, - { - "time": 1570514820, - "value": 293.747 - }, - { - "time": 1570514880, - "value": 335.68024507042253 - }, - { - "time": 1570514940, - "value": 302.6367777777778 - }, - { - "time": 1570515000, - "value": 293.5828404327248 - }, - { - "time": 1570515060, - "value": 364.85357731958766 - }, - { - "time": 1570515120, - "value": 299.2711410628019 - }, - { - "time": 1570515180, - "value": 250.0756365914787 - }, - { - "time": 1570515240, - "value": 238.3110868571429 - }, - { - "time": 1570515300, - "value": 238.61323474178403 - }, - { - "time": 1570515360, - "value": 170.26170370370372 - }, - { - "time": 1570515420, - "value": 308.52771810542396 - }, - { - "time": 1570515480, - "value": 289.1951803921569 - }, - { - "time": 1570515540, - "value": 196.91856255468073 - }, - { - "time": 1570515600, - "value": 304.4244072992701 - }, - { - "time": 1570515660, - "value": 363.43842683982683 - }, - { - "time": 1570515720, - "value": 330.26129216300944 - }, - { - "time": 1570515780, - "value": 285.3583933933934 - }, - { - "time": 1570515840, - "value": 337.3101340966726 - }, - { - "time": 1570515900, - "value": 340.99938260869567 - }, - { - "time": 1570515960, - "value": 310.56009677419354 - }, - { - "time": 1570516020, - "value": 308.1902492492493 - }, - { - "time": 1570516080, - "value": 301.68493514088254 - }, - { - "time": 1570516140, - "value": 345.33895635134087 - }, - { - "time": 1570516200, - "value": 315.150374863388 - }, - { - "time": 1570516260, - "value": 335.61720155038756 - }, - { - "time": 1570516320, - "value": 288.970167816092 - }, - { - "time": 1570516380, - "value": 347.7951777777778 - }, - { - "time": 1570516440, - "value": 278.4351945288754 - }, - { - "time": 1570516500, - "value": 311.5431904761905 - }, - { - "time": 1570516560, - "value": 345.89566666666667 - }, - { - "time": 1570516620, - "value": 302.80535294117647 - }, - { - "time": 1570516680, - "value": 288.29416615698267 - }, - { - "time": 1570516740, - "value": 256.9934761904762 - }, - { - "time": 1570516800, - "value": 274.1354156028369 - }, - { - "time": 1570516860, - "value": 225.67473333333334 - }, - { - "time": 1570516920, - "value": 332.3788666666667 - }, - { - "time": 1570516980, - "value": 295.00328539199097 - }, - { - "time": 1570517040, - "value": 299.80455897435894 - }, - { - "time": 1570517100, - "value": 293.97577655677657 - }, - { - "time": 1570517160, - "value": 291.5642604444444 - }, - { - "time": 1570517220, - "value": 315.1292142286171 - }, - { - "time": 1570517280, - "value": 315.99385714285717 - }, - { - "time": 1570517340, - "value": 268.4086462809917 - }, - { - "time": 1570517400, - "value": 230.69465714285718 - }, - { - "time": 1570517460, - "value": 204.20621975147154 - }, - { - "time": 1570517520, - "value": 268.33346690909093 - }, - { - "time": 1570517580, - "value": 394.0388280922432 - }, - { - "time": 1570517640, - "value": 266.4535437788018 - }, - { - "time": 1570517700, - "value": 234.24486028708137 - }, - { - "time": 1570517760, - "value": 240.50934829931973 - }, - { - "time": 1570517820, - "value": 265.6087622377622 - }, - { - "time": 1570517880, - "value": 285.217661728395 - }, - { - "time": 1570517940, - "value": 330.24400619094166 - }, - { - "time": 1570518000, - "value": 248.0476796992482 - }, - { - "time": 1570518060, - "value": 324.69353333333333 - }, - { - "time": 1570518120, - "value": 230.03679487179488 - }, - { - "time": 1570518180, - "value": 312.634794032994 - }, - { - "time": 1570518240, - "value": 246.6245873015873 - }, - { - "time": 1570518300, - "value": 382.1588900584795 - }, - { - "time": 1570518360, - "value": 279.6099894736842 - }, - { - "time": 1570518420, - "value": 254.31687559808617 - }, - { - "time": 1570518480, - "value": 279.3255915966387 - }, - { - "time": 1570518540, - "value": 343.4059344388147 - }, - { - "time": 1570518600, - "value": 227.30593706293706 - }, - { - "time": 1570518660, - "value": 163.93439024390244 - }, - { - "time": 1570518720, - "value": 410.6014857881137 - }, - { - "time": 1570518780, - "value": 331.9343681864235 - }, - { - "time": 1570518840, - "value": 199.2816600985222 - }, - { - "time": 1570518900, - "value": 263.328387755102 - }, - { - "time": 1570518960, - "value": 287.09224210526315 - }, - { - "time": 1570519020, - "value": 240.47300747198008 - }, - { - "time": 1570519080, - "value": 360.6038853228963 - }, - { - "time": 1570519140, - "value": 347.1667480713875 - }, - { - "time": 1570519200, - "value": 300.24195022624434 - }, - { - "time": 1570519260, - "value": 151.76594071146246 - }, - { - "time": 1570519320, - "value": 310.08171469411093 - }, - { - "time": 1570519380, - "value": 392.1943797101449 - }, - { - "time": 1570519440, - "value": 314.64166147500794 - }, - { - "time": 1570519500, - "value": 324.985020338983 - }, - { - "time": 1570519560, - "value": 339.2699803174604 - }, - { - "time": 1570519620, - "value": 364.1995050505051 - }, - { - "time": 1570519680, - "value": 322.0562205471803 - }, - { - "time": 1570519740, - "value": 336.0596713286714 - }, - { - "time": 1570519800, - "value": 192.54439684978874 - }, - { - "time": 1570519860, - "value": 379.64561356932154 - }, - { - "time": 1570519920, - "value": 283.12156161616167 - }, - { - "time": 1570519980, - "value": 153.6972603773585 - }, - { - "time": 1570520040, - "value": 297.7845619047619 - }, - { - "time": 1570520100, - "value": 251.20608034188032 - }, - { - "time": 1570520160, - "value": 238.4791967213115 - }, - { - "time": 1570520220, - "value": 275.54378123167163 - }, - { - "time": 1570520280, - "value": 264.39154945054946 - }, - { - "time": 1570520340, - "value": 243.7744285714286 - }, - { - "time": 1570520400, - "value": 313.15596 - }, - { - "time": 1570520460, - "value": 208.91436363636365 - }, - { - "time": 1570520520, - "value": 302.5077115188583 - }, - { - "time": 1570520580, - "value": 223.00830703986426 - }, - { - "time": 1570520640, - "value": 277.33216239316243 - }, - { - "time": 1570520700, - "value": 328.59207076923076 - }, - { - "time": 1570520760, - "value": 412.18899999999996 - }, - { - "time": 1570520820, - "value": 264.2438407982262 - }, - { - "time": 1570520880, - "value": 232.92365134099617 - }, - { - "time": 1570520940, - "value": 331.73893773803616 - }, - { - "time": 1570521000, - "value": 254.89367538411247 - }, - { - "time": 1570521060, - "value": 384.51859700093723 - }, - { - "time": 1570521120, - "value": 232.05635172413793 - }, - { - "time": 1570521180, - "value": 381.29816168582374 - }, - { - "time": 1570521240, - "value": 268.6763086660175 - }, - { - "time": 1570521300, - "value": 198.4742195121951 - }, - { - "time": 1570521360, - "value": 279.83657020757016 - }, - { - "time": 1570521420, - "value": 329.77706060606056 - }, - { - "time": 1570521480, - "value": 298.0864917874396 - }, - { - "time": 1570521540, - "value": 309.87589149560114 - }, - { - "time": 1570521600, - "value": 533.1009869281046 - }, - { - "time": 1570521660, - "value": 640.9424236535398 - }, - { - "time": 1570521720, - "value": 587.5921240795599 - }, - { - "time": 1570521780, - "value": 740.6469632277835 - }, - { - "time": 1570521840, - "value": 590.1231481420687 - }, - { - "time": 1570521900, - "value": 824.2047327295501 - }, - { - "time": 1570521960, - "value": 547.2808538699691 - }, - { - "time": 1570522020, - "value": 478.2387710046767 - }, - { - "time": 1570522080, - "value": 633.6360220913108 - }, - { - "time": 1570522140, - "value": 572.6200555830447 - }, - { - "time": 1570522200, - "value": 609.2879316261203 - }, - { - "time": 1570522260, - "value": 604.7186673496192 - }, - { - "time": 1570522320, - "value": 583.2839171717171 - }, - { - "time": 1570522380, - "value": 544.7179341614907 - }, - { - "time": 1570522440, - "value": 452.0116457614403 - }, - { - "time": 1570522500, - "value": 410.8066179885156 - }, - { - "time": 1570522560, - "value": 648.8380444734714 - }, - { - "time": 1570522620, - "value": 1030.2163382269905 - }, - { - "time": 1570522680, - "value": 768.3124568144499 - }, - { - "time": 1570522740, - "value": 411.8386285434995 - }, - { - "time": 1570522800, - "value": 556.0043582684721 - }, - { - "time": 1570522860, - "value": 619.8103673545966 - }, - { - "time": 1570522920, - "value": 437.01229729729727 - }, - { - "time": 1570522980, - "value": 652.7276058573665 - }, - { - "time": 1570523040, - "value": 496.62427115363226 - }, - { - "time": 1570523100, - "value": 625.6678648273469 - }, - { - "time": 1570523160, - "value": 647.0540419399117 - }, - { - "time": 1570523220, - "value": 594.6064396270397 - }, - { - "time": 1570523280, - "value": 602.7548548752835 - }, - { - "time": 1570523340, - "value": 499.4494646464647 - }, - { - "time": 1570523400, - "value": 480.08868503462804 - }, - { - "time": 1570523460, - "value": 491.0980092503987 - }, - { - "time": 1570523520, - "value": 351.9981936041486 - }, - { - "time": 1570523580, - "value": 489.7407263927435 - }, - { - "time": 1570523640, - "value": 594.4821717602464 - }, - { - "time": 1570523700, - "value": 641.7007912960306 - }, - { - "time": 1570523760, - "value": 494.5754713211601 - }, - { - "time": 1570523820, - "value": 843.1369691297209 - }, - { - "time": 1570523880, - "value": 607.5716861788618 - }, - { - "time": 1570523940, - "value": 570.5909637159838 - }, - { - "time": 1570524000, - "value": 628.2413762849629 - }, - { - "time": 1570524060, - "value": 547.4951196581197 - }, - { - "time": 1570524120, - "value": 618.319142995169 - }, - { - "time": 1570524180, - "value": 808.1159647413266 - }, - { - "time": 1570524240, - "value": 644.3792774723114 - }, - { - "time": 1570524300, - "value": 543.6315571448107 - }, - { - "time": 1570524360, - "value": 565.9620483495 - }, - { - "time": 1570524420, - "value": 518.5502519480519 - }, - { - "time": 1570524480, - "value": 449.71044957264957 - }, - { - "time": 1570524540, - "value": 572.4358950592078 - }, - { - "time": 1570524600, - "value": 549.5697692307692 - } - ], - "query": { - "tags_search": "*", - "metric": "metric1" - }, - "metric": { - "name": "metric1", - "type": "gauge", - "attributes": { - "display_units_long": "Milliseconds", - "created_by_ua": "Go-http-client/2.0", - "display_units_short": "ms", - "summarize_function": "average", - "aggregate": true - }, - "period": 60 - }, - "period": 60 - } - ], - "compose": "sum(s(\"metric1\", \"*\", {period: \"60\"}))", - "resolution": 60 -} \ No newline at end of file diff --git a/timeseriesql/tests/query_test.py b/timeseriesql/tests/query_test.py index dc97588..12b7e26 100644 --- a/timeseriesql/tests/query_test.py +++ b/timeseriesql/tests/query_test.py @@ -52,3 +52,6 @@ def test_iteration(self): for i, _ in enumerate(q): pass self.assertEqual(0, i) + +if __name__ == '__main__': + unittest.main() diff --git a/timeseriesql/tests/timeseries_test.py b/timeseriesql/tests/timeseries_test.py index 10b4d1f..d9a0c33 100644 --- a/timeseriesql/tests/timeseries_test.py +++ b/timeseriesql/tests/timeseries_test.py @@ -551,4 +551,7 @@ def test_nan_processing(self): combined = t1.merge([t2,t3]) combined[1::2,0] = np.nan self.assertEqual(4.4, np.nanmean(combined)) - self.assertEqual(combined.nanmean(), np.nanmean(combined)) \ No newline at end of file + self.assertEqual(combined.nanmean(), np.nanmean(combined)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/timeseriesql/timeseries.py b/timeseriesql/timeseries.py index 02dbd4c..ce20626 100644 --- a/timeseriesql/timeseries.py +++ b/timeseriesql/timeseries.py @@ -6,6 +6,7 @@ from itertools import compress from .timeseries_collection import TimeSeriesCollection from .time_chunk import TimeChunk +from .utils import dispatchmethod class TimeSeries: @@ -44,135 +45,6 @@ def __init__(self, *args, shape=(4, 3), labels=None, time=[], **kwargs): time = time[-self.data.shape[0]:] self._time = np.array(time, dtype=np.float64) - def __truediv__(self, other): - return self.data.__truediv__(other) - - def __add__(self, other): - return self.data.__add__(other) - - def __sub__(self, other): # pragma: no cover - return self.data.__sub__(other) - - def __mul__(self, other): - return self.data.__mul__(other) - - def __floordiv__(self, other):# pragma: no cover - return self.data.__floordiv__(other) - - def __mod__(self, other):# pragma: no cover - return self.data.__mod__(other) - - def __pow__(self, other):# pragma: no cover - return self.data.__pow__(other) - - def __lshift__(self, other):# pragma: no cover - return self.data.__lshift__(other) - - def __rshift__(self, other):# pragma: no cover - return self.data.__rshift__(other) - - def __and__(self, other):# pragma: no cover - return self.data.__and__(other) - - def __xor__(self, other):# pragma: no cover - return self.data.__xor__(other) - - def __or__(self, other):# pragma: no cover - return self.data.__or__(other) - - def __iadd__(self, other): - return self.data.__iadd__(other) - - def __isub__(self, other):# pragma: no cover - return self.data.__isub__(other) - - def __imul__(self, other): - return self.data.__imul__(other) - - def __idiv__(self, other):# pragma: no cover - return self.data.__idiv__(other) - - def __ifloordiv__(self, other):# pragma: no cover - return self.data.__ifloordiv__(other) - - def __imod__(self, other):# pragma: no cover - return self.data.__imod__(other) - - def __ipow__(self, other):# pragma: no cover - return self.data.__ipow__(other) - - def __ilshift__(self, other):# pragma: no cover - return self.data.__ilshift__(other) - - def __irshift__(self, other):# pragma: no cover - return self.data.__irshift__(other) - - def __iand__(self, other):# pragma: no cover - return self.data.__iand__(other) - - def __ixor__(self, other):# pragma: no cover - return self.data.__ixor__(other) - - def __ior__(self, other):# pragma: no cover - return self.data.__ior__(other) - - def __radd__(self, other):# pragma: no cover - return self.data.__radd__(other) - - def __rsub__(self, other):# pragma: no cover - return self.data.__rsub__(other) - - def __rmul__(self, other): - return self.data.__rmul__(other) - - def __rdiv__(self, other):# pragma: no cover - return self.data.__rdiv__(other) - - def __rfloordiv__(self, other):# pragma: no cover - return self.data.__rfloordiv__(other) - - def __rmod__(self, other):# pragma: no cover - return self.data.__rmod__(other) - - def __rpow__(self, other):# pragma: no cover - return self.data.__rpow__(other) - - def __rlshift__(self, other):# pragma: no cover - return self.data.__rlshift__(other) - - def __rrshift__(self, other):# pragma: no cover - return self.data.__rrshift__(other) - - def __rand__(self, other):# pragma: no cover - return self.data.__rand__(other) - - def __rxor__(self, other):# pragma: no cover - return self.data.__rxor__(other) - - def __ror__(self, other):# pragma: no cover - return self.data.__ror__(other) - - def __lt__(self, other):# pragma: no cover - return self.data.__lt__(other) - - def __le__(self, other):# pragma: no cover - return self.data.__le__(other) - - def __eq__(self, other): - return self.data.__eq__(other) - - def __ne__(self, other):# pragma: no cover - return self.data.__ne__(other) - - def __gt__(self, other):# pragma: no cover - return self.data.__gt__(other) - - def __ge__(self, other):# pragma: no cover - return self.data.__ge__(other) - - def __len__(self): - return self.data.__len__() - def __array__(self):# pragma: no cover return self.data @@ -235,22 +107,8 @@ def __setitem__(self, key, item): except TypeError as e: return self.data.__setitem__(self._handle_type_error(key, e), item) - + @dispatchmethod def _handle_index_error(self, key, error): - time_slice = key - extra_slices = None - if isinstance(key, tuple): - time_slice, extra_slices = key - if isinstance(time_slice, (np.datetime64, slice)): - start, stop = self._get_slice_by_datetime(time_slice) - if (start, stop) == (None, None): - raise error - if extra_slices: - return (start, extra_slices) - else: - return start - if isinstance(key, int): - raise error l = len(key) if l != 1: raise error @@ -260,6 +118,25 @@ def _handle_index_error(self, key, error): return slice(0,0) # maybe should return something else? return (slice(None), filters) + @_handle_index_error.register(int) + def _(self, key, error): + raise error + + @_handle_index_error.register(tuple) + @_handle_index_error.register(slice) + @_handle_index_error.register(np.datetime64) + def _(self, key, error): + time_slice = key + extra_slices = None + if isinstance(key, tuple): + time_slice, extra_slices = key + start, stop = self._get_slice_by_datetime(time_slice) + if (start, stop) == (None, None): + raise error + if extra_slices: + return (start, extra_slices) + return start + def _handle_type_error(self, key, error, item = None): # build new slice time_slice = key @@ -357,14 +234,6 @@ def _least_common_labels(self, mask): del common_labels[k] return common_labels - def _generate_title(self): - """Generate a title based on common tags""" - # get common labels - labels = self._get_unique_keys() - if len(labels) > 0: - return ".".join([str(v) for k, v in labels.items()]) - return "TimeSeries" - def _get_unique_keys(self): """Get list of unique keys""" labels = self.labels[0].copy() @@ -374,13 +243,6 @@ def _get_unique_keys(self): del labels[k] return labels - def _generate_labels(self): - common_labels = self._get_unique_keys().keys() - labels = [] - for l in self.labels: - labels.append(".".join([str(v) for k, v in l.items() if k not in common_labels])) - return labels - @property def time(self): """Return only the TimeSeries index""" @@ -691,4 +553,19 @@ def copy(self): """ ts = TimeSeries(shape=self.shape, labels=self.labels.copy(), time=self._time) ts.data = self.data.copy() - return ts \ No newline at end of file + return ts + + +def delegate(cls, attr_name, method_name): + def delegated(self, *vargs, **kwargs): + a = getattr(self, attr_name) + m = getattr(a, method_name) + return m(*vargs, **kwargs) + setattr(cls, method_name, delegated) + +dunder_funcs = ['truediv', 'add', 'sub', 'mul', 'floordiv', 'mod', 'pow', 'lshift', 'rshift', 'and', 'xor', 'or', 'iadd', 'isub', 'imul', 'idiv', 'ifloordiv', 'imod', 'ipow', 'ilshift', 'irshift', 'iand', 'ixor', 'radd', 'rsub', 'rmul', 'rdiv', 'rfloordiv', 'rmod', 'rpow', 'rlshift', 'rand', 'rxor', 'ror', 'lt', 'le', 'ne', 'eq', 'gt', 'ge','len'] + +# proxy the magic methods to the numpy array +for name in dunder_funcs: + name = f"__{name}__" + delegate(TimeSeries, 'data', name) \ No newline at end of file diff --git a/timeseriesql/utils/__init__.py b/timeseriesql/utils/__init__.py index d3f4517..6f2aa00 100644 --- a/timeseriesql/utils/__init__.py +++ b/timeseriesql/utils/__init__.py @@ -2,3 +2,93 @@ # This sub-module is destined for common non-package specific utility # functions. + + +# https://gist.github.com/adamnew123456/9218f99ba35da225ca11 +# this can be replaced https://docs.python.org/3/library/functools.html#functools.singledispatchmethod +# but it is only available in 3.8+ +from collections import namedtuple +from functools import singledispatch, update_wrapper + + +def dispatchmethod(func): + """ + This provides for a way to use ``functools.singledispatch`` inside of a class. It has the same + basic interface that ``singledispatch`` does: + + >>> class A: + ... @dispatchmethod + ... def handle_message(self, message): + ... # Fallback code... + ... pass + ... @handle_message.register(int) + ... def _(self, message): + ... # Special int handling code... + ... pass + ... + >>> a = A() + >>> a.handle_message(42) + # Runs "Special int handling code..." + + Note that using ``singledispatch`` in these cases is impossible, since it tries to dispatch + on the ``self`` argument, not the ``message`` argument. This is technically a double + dispatch, since both the type of ``self`` and the type of the second argument are used to + determine what function to call - for example: + + >>> class A: + ... @dispatchmethod + ... def handle_message(self, message): + ... print('A other', message) + ... pass + ... @handle_message.register(int) + ... def _(self, message): + ... print('A int', message) + ... pass + ... + >>> class B: + ... @dispatchmethod + ... def handle_message(self, message): + ... print('B other', message) + ... pass + ... @handle_message.register(int) + ... def _(self, message): + ... print('B int', message) + ... + >>> def do_stuff(A_or_B): + ... A_or_B.handle_message(42) + ... A_or_B.handle_message('not an int') + + On one hand, either the ``dispatchmethod`` defined in ``A`` or ``B`` is used depending + upon what object one passes to ``do_stuff()``, but on the other hand, ``do_stuff()`` + causes different versions of the dispatchmethod (found in either ``A`` or ``B``) + to be called (both the fallback and the ``int`` versions are implicitly called). + + Note that this should be fully compatable with ``singledispatch`` in any other respects + (that is, it exposes the same attributes and methods). + """ + dispatcher = singledispatch(func) + + def register(type, func=None): + if func is not None: + return dispatcher.register(type, func) + else: + + def _register(func): + return dispatcher.register(type)(func) + + return _register + + def dispatch(type): + return dispatcher.dispatch(type) + + def wrapper(inst, dispatch_data, *args, **kwargs): + cls = type(dispatch_data) + impl = dispatch(cls) + return impl(inst, dispatch_data, *args, **kwargs) + + wrapper.register = register + wrapper.dispatch = dispatch + wrapper.registry = dispatcher.registry + wrapper._clear_cache = dispatcher._clear_cache + update_wrapper(wrapper, func) + return wrapper