From 2418c561cc36b17875ac042597b7f9218c8b8908 Mon Sep 17 00:00:00 2001 From: Douglas Macdonald Date: Fri, 1 Jun 2018 14:12:38 +0100 Subject: [PATCH 1/6] Fix for mid date interpolation `steps[0, 1:-1:2] = steps[0, 2::2] = (x[:-1] + x[1:]) / 2` This does not work for interpolating datetimes. This is because adding date does not makes sence and Numpy will through an error `TypeError: ufunc add cannot use operands with types dtype(' Date: Fri, 15 Jun 2018 14:21:22 +0100 Subject: [PATCH 2/6] Improved handling of dates in interpolate_curve --- holoviews/operation/element.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/holoviews/operation/element.py b/holoviews/operation/element.py index 368821a68d..938154695a 100644 --- a/holoviews/operation/element.py +++ b/holoviews/operation/element.py @@ -680,25 +680,17 @@ def pts_to_prestep(cls, x, y): steps[0, 1::2] = steps[0, 0:-2:2] steps[1:, 0::2] = y steps[1:, 1::2] = steps[1:, 2::2] - return steps + return steps[0, :], steps[1, :] @classmethod def pts_to_midstep(cls, x, y): steps = np.zeros((2, 2 * len(x))) - x = np.asanyarray(x) - - # For this to work for dates as well as numbers this must be in the form - # t1 + (t2 - t1)/2 = t1 + time_delta - # and NOT - # (t1 + t2)/2 = date + date Addind dates is an error. - # This is somewhat fragile and might be replaced with a funciton call to - # contain the calculation. steps[0, 1:-1:2] = steps[0, 2::2] = x[:-1] + (x[1:] - x[:-1])/2 steps[0, 0], steps[0, -1] = x[0], x[-1] steps[1:, 0::2] = y steps[1:, 1::2] = steps[1:, 0::2] - return steps + return steps[0, :], steps[1, :] @classmethod def pts_to_poststep(cls, x, y): @@ -707,7 +699,7 @@ def pts_to_poststep(cls, x, y): steps[0, 1::2] = steps[0, 2::2] steps[1:, 0::2] = y steps[1:, 1::2] = steps[1:, 0:-2:2] - return steps + return steps[0, :], steps[1, :] def _process_layer(self, element, key=None): INTERPOLATE_FUNCS = {'steps-pre': self.pts_to_prestep, @@ -716,11 +708,14 @@ def _process_layer(self, element, key=None): if self.p.interpolation not in INTERPOLATE_FUNCS: return element x, y = element.dimension_values(0), element.dimension_values(1) - if 'f' in (x.dtype.kind, y.dtype.kind): - x, y = x.astype('float'), y.astype('float') - array = INTERPOLATE_FUNCS[self.p.interpolation](x, y) + is_datetime = x.dtype.kind == 'M' or isinstance(x[0], datetime_types) + if is_datetime: + x = x.astype('datetime64[ns]').astype('int64') * 1000. + xs, ys = INTERPOLATE_FUNCS[self.p.interpolation](x.astype('f'), y.astype('f')) + if is_datetime: + xs = (xs/10e5).astype('datetime64[us]') dvals = tuple(element.dimension_values(d) for d in element.dimensions()[2:]) - return element.clone((array[0, :].astype(x.dtype), array[1, :].astype(y.dtype))+dvals) + return element.clone((xs, ys.astype(y.dtype))+dvals) def _process(self, element, key=None): return element.map(self._process_layer, Element) From 4fb1b018fba07eb96b76dc59effe6f49efa5b3d9 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 15 Jun 2018 14:21:34 +0100 Subject: [PATCH 3/6] Fixed array comparisons for datetimes --- holoviews/element/comparison.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/holoviews/element/comparison.py b/holoviews/element/comparison.py index b7afb1cb61..487ccb2cb4 100644 --- a/holoviews/element/comparison.py +++ b/holoviews/element/comparison.py @@ -251,6 +251,10 @@ def compare_floats(cls, arr1, arr2, msg='Floats'): @classmethod def compare_arrays(cls, arr1, arr2, msg='Arrays'): try: + if arr1.dtype.kind == 'M': + arr1 = arr1.astype('datetime64[ns]').astype('int64') + if arr2.dtype.kind == 'M': + arr2 = arr2.astype('datetime64[ns]').astype('int64') assert_array_equal(arr1, arr2) except: try: From ac1c9c90aefeb5dcce778d715bc76ec95a671c6b Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 15 Jun 2018 14:21:49 +0100 Subject: [PATCH 4/6] Added unit tests for interpolate_curve with dates --- tests/operation/testoperation.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/operation/testoperation.py b/tests/operation/testoperation.py index 7b59174b77..8aaacda0c1 100644 --- a/tests/operation/testoperation.py +++ b/tests/operation/testoperation.py @@ -192,16 +192,56 @@ def test_interpolate_curve_pre(self): curve = Curve([(0, 0), (0, 0.5), (1, 0.5), (1, 1), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_pre(self): + dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') + values = [0, 1, 2, 3] + interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-pre') + dates_interp = np.array([ + '2016-12-31T23:58:50.602104000', '2016-12-31T23:58:50.602104000', + '2017-01-01T23:59:03.419954000', '2017-01-01T23:59:03.419954000', + '2017-01-02T23:59:16.237805000', '2017-01-02T23:59:16.237805000', + '2017-01-03T23:59:29.055655000' + ], dtype='datetime64[ns]') + print(interpolated.data.dtypes) + curve = Curve((dates_interp, [0, 1, 1, 2, 2, 3, 3])) + self.assertEqual(interpolated, curve) + def test_interpolate_curve_mid(self): interpolated = interpolate_curve(Curve([0, 0.5, 1]), interpolation='steps-mid') curve = Curve([(0, 0), (0.5, 0), (0.5, 0.5), (1.5, 0.5), (1.5, 1), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_mid(self): + dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') + values = [0, 1, 2, 3] + interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-mid') + dates_interp = np.array([ + '2016-12-31T23:58:50.602104000', '2017-01-01T11:58:57.011029000', + '2017-01-01T11:58:57.011029000', '2017-01-02T11:59:09.828879000', + '2017-01-02T11:59:09.828879000', '2017-01-03T11:59:22.646730000', + '2017-01-03T11:59:22.646730000', '2017-01-03T23:59:29.055655000' + ], dtype='datetime64[ns]') + curve = Curve((dates_interp, [0, 0, 1, 1, 2, 2, 3, 3])) + self.assertEqual(interpolated, curve) + def test_interpolate_curve_post(self): interpolated = interpolate_curve(Curve([0, 0.5, 1]), interpolation='steps-post') curve = Curve([(0, 0), (1, 0), (1, 0.5), (2, 0.5), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_post(self): + dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') + values = [0, 1, 2, 3] + interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-post') + dates_interp = np.array([ + '2016-12-31T23:58:50.602104000', '2017-01-01T23:59:03.419954000', + '2017-01-01T23:59:03.419954000', '2017-01-02T23:59:16.237805000', + '2017-01-02T23:59:16.237805000', '2017-01-03T23:59:29.055655000', + '2017-01-03T23:59:29.055655000' + ], dtype='datetime64[ns]') + curve = Curve((dates_interp, [0, 0, 1, 1, 2, 2, 3])) + self.assertEqual(interpolated, curve) + def test_stack_area_overlay(self): areas = Area([1, 2, 3]) * Area([1, 2, 3]) stacked = Area.stack(areas) From 986659767c914b98e9092a510f7ea7f3dbe798d0 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 15 Jun 2018 14:51:23 +0100 Subject: [PATCH 5/6] Further cleaned up interpolate_curve datetime handling --- holoviews/operation/element.py | 8 +++++--- tests/operation/testoperation.py | 25 ++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/holoviews/operation/element.py b/holoviews/operation/element.py index 938154695a..c316c5a33b 100644 --- a/holoviews/operation/element.py +++ b/holoviews/operation/element.py @@ -708,12 +708,14 @@ def _process_layer(self, element, key=None): if self.p.interpolation not in INTERPOLATE_FUNCS: return element x, y = element.dimension_values(0), element.dimension_values(1) - is_datetime = x.dtype.kind == 'M' or isinstance(x[0], datetime_types) + dtype = x.dtype + is_datetime = dtype.kind == 'M' or isinstance(x[0], datetime_types) if is_datetime: - x = x.astype('datetime64[ns]').astype('int64') * 1000. + dt_type = dtype if dtype.kind == 'M' else 'datetime64[ns]' + x = x.astype(dt_type).astype('int64') xs, ys = INTERPOLATE_FUNCS[self.p.interpolation](x.astype('f'), y.astype('f')) if is_datetime: - xs = (xs/10e5).astype('datetime64[us]') + xs = xs.astype(dt_type) dvals = tuple(element.dimension_values(d) for d in element.dimensions()[2:]) return element.clone((xs, ys.astype(y.dtype))+dvals) diff --git a/tests/operation/testoperation.py b/tests/operation/testoperation.py index 8aaacda0c1..5b39899254 100644 --- a/tests/operation/testoperation.py +++ b/tests/operation/testoperation.py @@ -197,12 +197,11 @@ def test_interpolate_datetime_curve_pre(self): values = [0, 1, 2, 3] interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-pre') dates_interp = np.array([ - '2016-12-31T23:58:50.602104000', '2016-12-31T23:58:50.602104000', - '2017-01-01T23:59:03.419954000', '2017-01-01T23:59:03.419954000', - '2017-01-02T23:59:16.237805000', '2017-01-02T23:59:16.237805000', - '2017-01-03T23:59:29.055655000' + '2017-01-01T00:00:16.364011520', '2017-01-01T00:00:16.364011520', + '2017-01-02T00:01:05.465745408', '2017-01-02T00:01:05.465745408', + '2017-01-02T23:59:37.128525824', '2017-01-02T23:59:37.128525824', + '2017-01-04T00:00:26.230259712' ], dtype='datetime64[ns]') - print(interpolated.data.dtypes) curve = Curve((dates_interp, [0, 1, 1, 2, 2, 3, 3])) self.assertEqual(interpolated, curve) @@ -216,10 +215,10 @@ def test_interpolate_datetime_curve_mid(self): values = [0, 1, 2, 3] interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-mid') dates_interp = np.array([ - '2016-12-31T23:58:50.602104000', '2017-01-01T11:58:57.011029000', - '2017-01-01T11:58:57.011029000', '2017-01-02T11:59:09.828879000', - '2017-01-02T11:59:09.828879000', '2017-01-03T11:59:22.646730000', - '2017-01-03T11:59:22.646730000', '2017-01-03T23:59:29.055655000' + '2017-01-01T00:00:16.364011520', '2017-01-01T11:59:32.195401728', + '2017-01-01T11:59:32.195401728', '2017-01-02T12:00:21.297135616', + '2017-01-02T12:00:21.297135616', '2017-01-03T12:01:10.398869504', + '2017-01-03T12:01:10.398869504', '2017-01-04T00:00:26.230259712' ], dtype='datetime64[ns]') curve = Curve((dates_interp, [0, 0, 1, 1, 2, 2, 3, 3])) self.assertEqual(interpolated, curve) @@ -234,10 +233,10 @@ def test_interpolate_datetime_curve_post(self): values = [0, 1, 2, 3] interpolated = interpolate_curve(Curve((dates, values)), interpolation='steps-post') dates_interp = np.array([ - '2016-12-31T23:58:50.602104000', '2017-01-01T23:59:03.419954000', - '2017-01-01T23:59:03.419954000', '2017-01-02T23:59:16.237805000', - '2017-01-02T23:59:16.237805000', '2017-01-03T23:59:29.055655000', - '2017-01-03T23:59:29.055655000' + '2017-01-01T00:00:16.364011520', '2017-01-02T00:01:05.465745408', + '2017-01-02T00:01:05.465745408', '2017-01-02T23:59:37.128525824', + '2017-01-02T23:59:37.128525824', '2017-01-04T00:00:26.230259712', + '2017-01-04T00:00:26.230259712' ], dtype='datetime64[ns]') curve = Curve((dates_interp, [0, 0, 1, 1, 2, 2, 3])) self.assertEqual(interpolated, curve) From 38d4722201f867e9cb4e7fbdf4290d9c954d787c Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 20 Jun 2018 14:19:17 +0100 Subject: [PATCH 6/6] Correctly handle additional value dimensions in Curve interpolation --- holoviews/operation/element.py | 70 ++++++++++++++++++++------------ tests/operation/testoperation.py | 22 ++++++++++ 2 files changed, 67 insertions(+), 25 deletions(-) diff --git a/holoviews/operation/element.py b/holoviews/operation/element.py index c316c5a33b..07cdf83073 100644 --- a/holoviews/operation/element.py +++ b/holoviews/operation/element.py @@ -674,32 +674,52 @@ class interpolate_curve(Operation): Controls the transition point of the step along the x-axis.""") @classmethod - def pts_to_prestep(cls, x, y): - steps = np.zeros((2, 2 * len(x) - 1)) - steps[0, 0::2] = x - steps[0, 1::2] = steps[0, 0:-2:2] - steps[1:, 0::2] = y - steps[1:, 1::2] = steps[1:, 2::2] - return steps[0, :], steps[1, :] + def pts_to_prestep(cls, x, values): + steps = np.zeros(2 * len(x) - 1) + value_steps = tuple(np.empty(2 * len(x) - 1, dtype=v.dtype) for v in values) + + steps[0::2] = x + steps[1::2] = steps[0:-2:2] + + val_arrays = [] + for v, s in zip(values, value_steps): + s[0::2] = v + s[1::2] = s[2::2] + val_arrays.append(s) + + return steps, tuple(val_arrays) @classmethod - def pts_to_midstep(cls, x, y): - steps = np.zeros((2, 2 * len(x))) - steps[0, 1:-1:2] = steps[0, 2::2] = x[:-1] + (x[1:] - x[:-1])/2 + def pts_to_midstep(cls, x, values): + steps = np.zeros(2 * len(x)) + value_steps = tuple(np.empty(2 * len(x), dtype=v.dtype) for v in values) + + steps[1:-1:2] = steps[2::2] = x[:-1] + (x[1:] - x[:-1])/2 + steps[0], steps[-1] = x[0], x[-1] - steps[0, 0], steps[0, -1] = x[0], x[-1] - steps[1:, 0::2] = y - steps[1:, 1::2] = steps[1:, 0::2] - return steps[0, :], steps[1, :] + val_arrays = [] + for v, s in zip(values, value_steps): + s[0::2] = v + s[1::2] = s[0::2] + val_arrays.append(s) + + return steps, tuple(val_arrays) @classmethod - def pts_to_poststep(cls, x, y): - steps = np.zeros((2, 2 * len(x) - 1)) - steps[0, 0::2] = x - steps[0, 1::2] = steps[0, 2::2] - steps[1:, 0::2] = y - steps[1:, 1::2] = steps[1:, 0:-2:2] - return steps[0, :], steps[1, :] + def pts_to_poststep(cls, x, values): + steps = np.zeros(2 * len(x) - 1) + value_steps = tuple(np.empty(2 * len(x) - 1, dtype=v.dtype) for v in values) + + steps[0::2] = x + steps[1::2] = steps[2::2] + + val_arrays = [] + for v, s in zip(values, value_steps): + s[0::2] = v + s[1::2] = s[0:-2:2] + val_arrays.append(s) + + return steps, tuple(val_arrays) def _process_layer(self, element, key=None): INTERPOLATE_FUNCS = {'steps-pre': self.pts_to_prestep, @@ -707,17 +727,17 @@ def _process_layer(self, element, key=None): 'steps-post': self.pts_to_poststep} if self.p.interpolation not in INTERPOLATE_FUNCS: return element - x, y = element.dimension_values(0), element.dimension_values(1) + x = element.dimension_values(0) dtype = x.dtype is_datetime = dtype.kind == 'M' or isinstance(x[0], datetime_types) if is_datetime: dt_type = dtype if dtype.kind == 'M' else 'datetime64[ns]' x = x.astype(dt_type).astype('int64') - xs, ys = INTERPOLATE_FUNCS[self.p.interpolation](x.astype('f'), y.astype('f')) + dvals = tuple(element.dimension_values(d) for d in element.dimensions()[1:]) + xs, dvals = INTERPOLATE_FUNCS[self.p.interpolation](x.astype('f'), dvals) if is_datetime: xs = xs.astype(dt_type) - dvals = tuple(element.dimension_values(d) for d in element.dimensions()[2:]) - return element.clone((xs, ys.astype(y.dtype))+dvals) + return element.clone((xs,)+dvals) def _process(self, element, key=None): return element.map(self._process_layer, Element) diff --git a/tests/operation/testoperation.py b/tests/operation/testoperation.py index 5b39899254..e1d709fec5 100644 --- a/tests/operation/testoperation.py +++ b/tests/operation/testoperation.py @@ -192,6 +192,12 @@ def test_interpolate_curve_pre(self): curve = Curve([(0, 0), (0, 0.5), (1, 0.5), (1, 1), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_curve_pre_with_values(self): + interpolated = interpolate_curve(Curve([(0, 0, 'A'), (1, 0.5, 'B'), (2, 1, 'C')], vdims=['y', 'z']), + interpolation='steps-pre') + curve = Curve([(0, 0, 'A'), (0, 0.5, 'B'), (1, 0.5, 'B'), (1, 1, 'C'), (2, 1, 'C')], vdims=['y', 'z']) + self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_pre(self): dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') values = [0, 1, 2, 3] @@ -210,6 +216,14 @@ def test_interpolate_curve_mid(self): curve = Curve([(0, 0), (0.5, 0), (0.5, 0.5), (1.5, 0.5), (1.5, 1), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_curve_mid_with_values(self): + interpolated = interpolate_curve(Curve([(0, 0, 'A'), (1, 0.5, 'B'), (2, 1, 'C')], vdims=['y', 'z']), + interpolation='steps-mid') + curve = Curve([(0, 0, 'A'), (0.5, 0, 'A'), (0.5, 0.5, 'B'), + (1.5, 0.5, 'B'), (1.5, 1, 'C'), (2, 1, 'C')], + vdims=['y', 'z']) + self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_mid(self): dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') values = [0, 1, 2, 3] @@ -228,6 +242,14 @@ def test_interpolate_curve_post(self): curve = Curve([(0, 0), (1, 0), (1, 0.5), (2, 0.5), (2, 1)]) self.assertEqual(interpolated, curve) + def test_interpolate_curve_post_with_values(self): + interpolated = interpolate_curve(Curve([(0, 0, 'A'), (1, 0.5, 'B'), (2, 1, 'C')], vdims=['y', 'z']), + interpolation='steps-post') + curve = Curve([(0, 0, 'A'), (1, 0, 'A'), (1, 0.5, 'B'), + (2, 0.5, 'B'), (2, 1, 'C')], + vdims=['y', 'z']) + self.assertEqual(interpolated, curve) + def test_interpolate_datetime_curve_post(self): dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') values = [0, 1, 2, 3]