From 043f7fcaabc72021f9256b21efec5bc979e9ef0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 8 Dec 2023 19:19:01 +0100 Subject: [PATCH 01/28] Add viewport downsample algorithm --- holoviews/operation/downsample.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index f7b2e75c82..cfd9d8911d 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -147,10 +147,14 @@ def _nth_point(x, y, n_out): n_samples = len(x) return np.arange(0, n_samples, max(1, math.ceil(n_samples / n_out))) +def _viewport(x, y, n_out): + return slice(0, len(x)) + _ALGORITHMS = { 'lttb': _lttb, - 'nth': _nth_point + 'nth': _nth_point, + 'viewport': _viewport, } @@ -164,7 +168,7 @@ class downsample1d(ResampleOperation1D): - `nth`: Selects every n-th point. """ - algorithm = param.Selector(default='lttb', objects=['lttb', 'nth']) + algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) def _process(self, element, key=None): if isinstance(element, (Overlay, NdOverlay)): @@ -176,7 +180,7 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - element = element[slice(*self.p.x_range)] + element = element[slice(self.p.x_range[0], self.p.x_range[1] + 1)] if len(element) <= self.p.width: return element xs, ys = (element.dimension_values(i) for i in range(2)) From bf428910a55dc05aa6588e2ebed64cc724984deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 09:56:43 +0100 Subject: [PATCH 02/28] pop non relevant stream ranges in RangeX and RangeY --- holoviews/streams.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/holoviews/streams.py b/holoviews/streams.py index 309b4ac153..75f66af7af 100644 --- a/holoviews/streams.py +++ b/holoviews/streams.py @@ -1480,6 +1480,10 @@ class RangeX(LinkedStream): x_range = param.Tuple(default=None, length=2, constant=True, doc=""" Range of the x-axis of a plot in data coordinates""") + def _set_stream_parameters(self, **kwargs): + kwargs.pop("y_range", None) + super()._set_stream_parameters(**kwargs) + class RangeY(LinkedStream): """ @@ -1489,6 +1493,10 @@ class RangeY(LinkedStream): y_range = param.Tuple(default=None, length=2, constant=True, doc=""" Range of the y-axis of a plot in data coordinates""") + def _set_stream_parameters(self, **kwargs): + kwargs.pop("x_range", None) + super()._set_stream_parameters(**kwargs) + class BoundsXY(LinkedStream): """ From 914197943fabe4699c925d02917988a554864e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 09:58:29 +0100 Subject: [PATCH 03/28] Rough implementation of finding first value outside of range --- holoviews/operation/downsample.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index cfd9d8911d..67b0b1ba71 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -180,7 +180,8 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - element = element[slice(self.p.x_range[0], self.p.x_range[1] + 1)] + # TODO: find a better way to get the first value outside the range + element = element[slice(0.95 * self.p.x_range[0], self.p.x_range[1] * 1.05)] if len(element) <= self.p.width: return element xs, ys = (element.dimension_values(i) for i in range(2)) From 0943eeae6ebae6bb48afc33f727988878bdea8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 09:58:42 +0100 Subject: [PATCH 04/28] Remove unnessary 0 --- holoviews/operation/downsample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 67b0b1ba71..6a3956a97f 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -148,7 +148,7 @@ def _nth_point(x, y, n_out): return np.arange(0, n_samples, max(1, math.ceil(n_samples / n_out))) def _viewport(x, y, n_out): - return slice(0, len(x)) + return slice(len(x)) _ALGORITHMS = { From 45d89daa2f97bdbaa66c3fe8c8e2fd1c4694195b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 09:58:56 +0100 Subject: [PATCH 05/28] Add hack to work with xlim --- holoviews/operation/downsample.py | 1 + 1 file changed, 1 insertion(+) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 6a3956a97f..98f9c2923d 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -171,6 +171,7 @@ class downsample1d(ResampleOperation1D): algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) def _process(self, element, key=None): + self.p.x_range = self.p.x_range or element.opts.get().kwargs.get("xlim") # hack if isinstance(element, (Overlay, NdOverlay)): _process = partial(self._process, key=key) if isinstance(element, Overlay): From b2311b6a0ebcd2946a122140f815618eb71a84ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 10:02:15 +0100 Subject: [PATCH 06/28] Use slice instead of np.arange for nth --- holoviews/operation/downsample.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 98f9c2923d..d9c68860ea 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -142,10 +142,10 @@ def _nth_point(x, y, n_out): y (np.ndarray): The y-values of the data. n_out (int): The number of output points. Returns: - np.array: The indexes of the selected datapoints. + slice: The slice of selected datapoints. """ n_samples = len(x) - return np.arange(0, n_samples, max(1, math.ceil(n_samples / n_out))) + return slice(0, n_samples, max(1, math.ceil(n_samples / n_out))) def _viewport(x, y, n_out): return slice(len(x)) From e0ffefdcf0574d24889a1fa23add18c723184e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 19 Dec 2023 10:23:12 +0100 Subject: [PATCH 07/28] Improve detection first values outside of viewport --- holoviews/operation/downsample.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index d9c68860ea..e5c575f247 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -181,8 +181,9 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - # TODO: find a better way to get the first value outside the range - element = element[slice(0.95 * self.p.x_range[0], self.p.x_range[1] * 1.05)] + xs = element.dimension_values(0) + x0, x1 = (np.argmin(np.abs(xs-coord)) for coord in self.p.x_range) + element = element.iloc[max(x0 - 1, 0):(x1 + 2)] if len(element) <= self.p.width: return element xs, ys = (element.dimension_values(i) for i in range(2)) From d033ca08825c5500ea3c4e7d7fa1810ed0b6d992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 09:38:46 +0100 Subject: [PATCH 08/28] Use boolean logic instead --- holoviews/operation/downsample.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index e5c575f247..6582665343 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -181,9 +181,12 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - xs = element.dimension_values(0) - x0, x1 = (np.argmin(np.abs(xs-coord)) for coord in self.p.x_range) - element = element.iloc[max(x0 - 1, 0):(x1 + 2)] + mask = element.dataset.interface.select_mask(element.dataset, {element.kdims[0]: self.p.x_range}) + extra = mask[1:] ^ mask[:-1] + mask[1:] |= extra + mask[:-1] |= extra + element = element[mask] + if len(element) <= self.p.width: return element xs, ys = (element.dimension_values(i) for i in range(2)) From fef1b73e36cba5c720af1282ffd591b07c6c9ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 11:11:34 +0100 Subject: [PATCH 09/28] Add select_mask_neighbor --- holoviews/core/data/cudf.py | 12 ++++++++++++ holoviews/core/data/dask.py | 12 ++++++++++++ holoviews/core/data/ibis.py | 12 ++++++++++++ holoviews/core/data/interface.py | 15 +++++++++++++++ holoviews/operation/downsample.py | 13 +++++++++---- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/holoviews/core/data/cudf.py b/holoviews/core/data/cudf.py index 36af1ac49e..455a177183 100644 --- a/holoviews/core/data/cudf.py +++ b/holoviews/core/data/cudf.py @@ -231,6 +231,18 @@ def select_mask(cls, dataset, selection): mask &= new_mask return mask + @classmethod + def _select_mask_neighbor(cls, dataset, selection): + """Runs select mask and expand the True values to include its neighbors + + Example + + select_mask = [False, False, True, True, False, False] + select_mask_neighbor = [False, True, True, True, True, False] + + """ + raise NotImplementedError + @classmethod def select(cls, dataset, selection_mask=None, **selection): df = dataset.data diff --git a/holoviews/core/data/dask.py b/holoviews/core/data/dask.py index 5b1160220b..b984947aa4 100644 --- a/holoviews/core/data/dask.py +++ b/holoviews/core/data/dask.py @@ -162,6 +162,18 @@ def select(cls, dataset, selection_mask=None, **selection): return df[dataset.vdims[0].name].compute().iloc[0] return df + @classmethod + def _select_mask_neighbor(cls, dataset, selection): + """Runs select mask and expand the True values to include its neighbors + + Example + + select_mask = [False, False, True, True, False, False] + select_mask_neighbor = [False, True, True, True, True, False] + + """ + raise NotImplementedError + @classmethod def groupby(cls, dataset, dimensions, container_type, group_type, **kwargs): index_dims = [dataset.get_dimension(d) for d in dimensions] diff --git a/holoviews/core/data/ibis.py b/holoviews/core/data/ibis.py index 691a55d26c..f904995d07 100644 --- a/holoviews/core/data/ibis.py +++ b/holoviews/core/data/ibis.py @@ -427,6 +427,18 @@ def select_mask(cls, dataset, selection): predicates.append(column == object) return predicates + @classmethod + def _select_mask_neighbor(cls, dataset, selection): + """Runs select mask and expand the True values to include its neighbors + + Example + + select_mask = [False, False, True, True, False, False] + select_mask_neighbor = [False, True, True, True, True, False] + + """ + raise NotImplementedError + @classmethod def sample(cls, dataset, samples=None): import ibis diff --git a/holoviews/core/data/interface.py b/holoviews/core/data/interface.py index 0a60552f9b..cdd3e151a2 100644 --- a/holoviews/core/data/interface.py +++ b/holoviews/core/data/interface.py @@ -377,6 +377,21 @@ def select_mask(cls, dataset, selection): mask &= index_mask return mask + @classmethod + def _select_mask_neighbor(cls, dataset, selection): + """Runs select mask and expand the True values to include its neighbors + + Example + + select_mask = [False, False, True, True, False, False] + select_mask_neighbor = [False, True, True, True, True, False] + + """ + mask = cls.select_mask(dataset, selection) + extra = mask[1:] ^ mask[:-1] + mask[1:] |= extra + mask[:-1] |= extra + return mask @classmethod def indexed(cls, dataset, selection): diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 6582665343..a863909faf 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -181,10 +181,15 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - mask = element.dataset.interface.select_mask(element.dataset, {element.kdims[0]: self.p.x_range}) - extra = mask[1:] ^ mask[:-1] - mask[1:] |= extra - mask[:-1] |= extra + try: + mask = element.dataset.interface._select_mask_neighbor( + element.dataset, {element.kdims[0]: self.p.x_range} + ) + except NotImplementedError: + mask = slice(*self.p.x_range) + except Exception: + self.param.warning("Could not apply neighbor mask to downsample1d.") + mask = slice(*self.p.x_range) element = element[mask] if len(element) <= self.p.width: From d226e6ff87a5c25f539b9ed5d034d69c56b3d341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 11:34:20 +0100 Subject: [PATCH 10/28] Add _select_mask_neighbor to dask interface --- holoviews/core/data/dask.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/holoviews/core/data/dask.py b/holoviews/core/data/dask.py index b984947aa4..50e1257287 100644 --- a/holoviews/core/data/dask.py +++ b/holoviews/core/data/dask.py @@ -154,6 +154,8 @@ def select_mask(cls, dataset, selection): def select(cls, dataset, selection_mask=None, **selection): df = dataset.data if selection_mask is not None: + if hasattr(df, "__array__"): + return df.loc[selection_mask] return df[selection_mask] selection_mask = cls.select_mask(dataset, selection) indexed = cls.indexed(dataset, selection) @@ -172,7 +174,12 @@ def _select_mask_neighbor(cls, dataset, selection): select_mask_neighbor = [False, True, True, True, True, False] """ - raise NotImplementedError + mask = cls.select_mask(dataset, selection) + mask = mask.to_dask_array().compute_chunk_sizes() + extra = (mask[1:] ^ mask[:-1]) + mask[1:] |= extra + mask[:-1] |= extra + return mask @classmethod def groupby(cls, dataset, dimensions, container_type, group_type, **kwargs): From ae35f9e48be55b64aea8253b5ea1f4c3947f431f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 11:34:46 +0100 Subject: [PATCH 11/28] Add error message to logging --- holoviews/operation/downsample.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index a863909faf..d1bd22d39e 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -187,8 +187,8 @@ def _process(self, element, key=None): ) except NotImplementedError: mask = slice(*self.p.x_range) - except Exception: - self.param.warning("Could not apply neighbor mask to downsample1d.") + except Exception as e: + self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") mask = slice(*self.p.x_range) element = element[mask] From 3ceaf73b559f8dc66ecd325210c66f0c8b263348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 11:55:41 +0100 Subject: [PATCH 12/28] Correct if statement in dask interface --- holoviews/core/data/dask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/holoviews/core/data/dask.py b/holoviews/core/data/dask.py index 50e1257287..9efc422ec0 100644 --- a/holoviews/core/data/dask.py +++ b/holoviews/core/data/dask.py @@ -154,7 +154,8 @@ def select_mask(cls, dataset, selection): def select(cls, dataset, selection_mask=None, **selection): df = dataset.data if selection_mask is not None: - if hasattr(df, "__array__"): + import dask.array as da + if isinstance(selection_mask, da.Array): return df.loc[selection_mask] return df[selection_mask] selection_mask = cls.select_mask(dataset, selection) From 7ef8c6c959a705439d98d1f4c2cd535662223547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 13:49:19 +0100 Subject: [PATCH 13/28] Remove xlim hack --- holoviews/operation/downsample.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index d1bd22d39e..b3a5b48ea5 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -171,7 +171,11 @@ class downsample1d(ResampleOperation1D): algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) def _process(self, element, key=None): - self.p.x_range = self.p.x_range or element.opts.get().kwargs.get("xlim") # hack + if not self.p.x_range: + # We don't want to send all the data to the frontend on the first pass + # if we have set xlims. Therefore we return the element without any data, + # this will render the plot and trigger a second pass with the x_range set. + return element[()] if isinstance(element, (Overlay, NdOverlay)): _process = partial(self._process, key=key) if isinstance(element, Overlay): From 5fc5e73373ab3669dafdaa187c7ffddd83c3e1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 13:55:01 +0100 Subject: [PATCH 14/28] Nit change --- holoviews/core/data/dask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/data/dask.py b/holoviews/core/data/dask.py index 9efc422ec0..ebffe3519d 100644 --- a/holoviews/core/data/dask.py +++ b/holoviews/core/data/dask.py @@ -177,7 +177,7 @@ def _select_mask_neighbor(cls, dataset, selection): """ mask = cls.select_mask(dataset, selection) mask = mask.to_dask_array().compute_chunk_sizes() - extra = (mask[1:] ^ mask[:-1]) + extra = mask[1:] ^ mask[:-1] mask[1:] |= extra mask[:-1] |= extra return mask From 662cc9088f9f0cc3119da88db16fd5a9b118e878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 14:10:12 +0100 Subject: [PATCH 15/28] Move x_range check down --- holoviews/operation/downsample.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index b3a5b48ea5..8bcbeba490 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -171,11 +171,6 @@ class downsample1d(ResampleOperation1D): algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) def _process(self, element, key=None): - if not self.p.x_range: - # We don't want to send all the data to the frontend on the first pass - # if we have set xlims. Therefore we return the element without any data, - # this will render the plot and trigger a second pass with the x_range set. - return element[()] if isinstance(element, (Overlay, NdOverlay)): _process = partial(self._process, key=key) if isinstance(element, Overlay): @@ -195,6 +190,11 @@ def _process(self, element, key=None): self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") mask = slice(*self.p.x_range) element = element[mask] + else: + # We don't want to send all the data to the frontend on the first pass + # if we have set xlims. Therefore we return the element without any data, + # this will render the plot and trigger a second pass with the x_range set. + return element[()] if len(element) <= self.p.width: return element From 811196614fd5914bdf65018d45311a77cf879963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 20 Dec 2023 15:08:07 +0100 Subject: [PATCH 16/28] Send empty plot for real this time --- holoviews/operation/downsample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 8bcbeba490..952a7d6c29 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -194,7 +194,7 @@ def _process(self, element, key=None): # We don't want to send all the data to the frontend on the first pass # if we have set xlims. Therefore we return the element without any data, # this will render the plot and trigger a second pass with the x_range set. - return element[()] + return element.clone(data=[]) if len(element) <= self.p.width: return element From 3c9d2f6d15e1cfa4e16214864fa677ac7a705d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 22 Dec 2023 09:54:59 +0100 Subject: [PATCH 17/28] Add viewport-xlim as workaround for now --- holoviews/operation/downsample.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 952a7d6c29..857aec34b4 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -155,6 +155,7 @@ def _viewport(x, y, n_out): 'lttb': _lttb, 'nth': _nth_point, 'viewport': _viewport, + 'viewport-xlim': _viewport, } @@ -190,10 +191,14 @@ def _process(self, element, key=None): self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") mask = slice(*self.p.x_range) element = element[mask] - else: + elif self.p.algorithm == "viewport-xlim": # We don't want to send all the data to the frontend on the first pass - # if we have set xlims. Therefore we return the element without any data, + # if we have set xlims. Therefore we send the element without any data, # this will render the plot and trigger a second pass with the x_range set. + # This will not work with the matplotlib backend, because it does not update + # the x_range after the first pass, but that is not a problem because + # the matplotlib backend with the viewport algorithm does not make any sense. + # This is not very elegant, but it works. return element.clone(data=[]) if len(element) <= self.p.width: From 6ba3b6f70581ac59c24d3d827692682318cd05d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 22 Dec 2023 10:27:36 +0100 Subject: [PATCH 18/28] Add width points to get a better approximation of the data in the y-direction --- holoviews/operation/downsample.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 857aec34b4..b36c78c52e 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -198,8 +198,15 @@ def _process(self, element, key=None): # This will not work with the matplotlib backend, because it does not update # the x_range after the first pass, but that is not a problem because # the matplotlib backend with the viewport algorithm does not make any sense. - # This is not very elegant, but it works. - return element.clone(data=[]) + # This is not very elegant. + size = element.dataset.shape[0] + mask1 = element.dataset.interface.select_mask( + element.dataset, {element.kdims[0]: slice(self.p.width // 2)}, + ) + mask2 = element.dataset.interface.select_mask( + element.dataset, {element.kdims[0]: slice(size - self.p.width // 2, size)}, + ) + element = element[mask1 | mask2] if len(element) <= self.p.width: return element From 06436c51ab7425964e92779d67b1e449e31e04f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 22 Dec 2023 11:03:06 +0100 Subject: [PATCH 19/28] Add unit test --- holoviews/tests/core/data/base.py | 6 ++++++ holoviews/tests/core/data/test_ibisinterface.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/holoviews/tests/core/data/base.py b/holoviews/tests/core/data/base.py index ed8121688e..34bd776c0f 100644 --- a/holoviews/tests/core/data/base.py +++ b/holoviews/tests/core/data/base.py @@ -858,6 +858,12 @@ def test_dataset_transform_add_ht(self): kdims=self.kdims, vdims=self.vdims+['combined']) self.assertEqual(transformed, expected) + def test_select_with_neighbor(self): + select = self.table.interface.select_mask(self.table.dataset, {"Weight": 18}) + select_neighbor = self.table.interface._select_mask_neighbor(self.table.dataset, dict(Weight=18)) + + np.testing.assert_almost_equal(select, [False, True, False]) + np.testing.assert_almost_equal(select_neighbor, [True, True, True]) class ScalarColumnTests: diff --git a/holoviews/tests/core/data/test_ibisinterface.py b/holoviews/tests/core/data/test_ibisinterface.py index 3f0b4d1773..2b488ec7a6 100644 --- a/holoviews/tests/core/data/test_ibisinterface.py +++ b/holoviews/tests/core/data/test_ibisinterface.py @@ -256,6 +256,13 @@ def test_aggregation_operations(self): self.compare_dataset(expected, result, msg=str(agg)) + def test_select_with_neighbor(self): + try: + # Not currently supported by Ibis + super().test_select_with_neighbor() + except NotImplementedError: + raise SkipTest("Not supported") + if not IbisInterface.has_rowid(): def test_dataset_iloc_slice_rows_slice_cols(self): From 9d6adc85b7f1df9a6b50f1d97fe3d31c6723cef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 22 Dec 2023 11:42:03 +0100 Subject: [PATCH 20/28] Add viewport options to docstring [skip ci] --- holoviews/operation/downsample.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index b36c78c52e..a63f5193e9 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -167,6 +167,8 @@ class downsample1d(ResampleOperation1D): - `lttb`: Largest Triangle Three Buckets downsample algorithm - `nth`: Selects every n-th point. + - `viewport`: Selects all points in a given viewport + - `viewport-xlim`: Selects all points in a given viewport, when xlim is set """ algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) From ee457ae7b370405c1aaec79c4ab0f86e1bb70814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Fri, 22 Dec 2023 12:03:03 +0100 Subject: [PATCH 21/28] Update comment [skip ci] --- holoviews/operation/downsample.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index a63f5193e9..cd495987be 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -194,12 +194,12 @@ def _process(self, element, key=None): mask = slice(*self.p.x_range) element = element[mask] elif self.p.algorithm == "viewport-xlim": - # We don't want to send all the data to the frontend on the first pass - # if we have set xlims. Therefore we send the element without any data, - # this will render the plot and trigger a second pass with the x_range set. - # This will not work with the matplotlib backend, because it does not update - # the x_range after the first pass, but that is not a problem because - # the matplotlib backend with the viewport algorithm does not make any sense. + # We only want to send some of the data to the browser on the + # first pass if we have set xlim. This will render the plot and + # trigger a second pass with the x_range set. This will not work + # with the matplotlib backend because it does not update the x_range + # after the first pass, but that is not a problem because the matplotlib + # backend with the viewport algorithm does not make any sense. # This is not very elegant. size = element.dataset.shape[0] mask1 = element.dataset.interface.select_mask( From 257e5fb6e276c9c0a00629005a2ca8a4ec2a5aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 17 Jan 2024 16:02:07 +0100 Subject: [PATCH 22/28] Only send the needed data during the initial rendering --- holoviews/operation/downsample.py | 19 ------------------- holoviews/plotting/bokeh/callbacks.py | 6 ++++++ holoviews/plotting/bokeh/plot.py | 3 --- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index cd495987be..269f3d92ac 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -155,7 +155,6 @@ def _viewport(x, y, n_out): 'lttb': _lttb, 'nth': _nth_point, 'viewport': _viewport, - 'viewport-xlim': _viewport, } @@ -168,7 +167,6 @@ class downsample1d(ResampleOperation1D): - `lttb`: Largest Triangle Three Buckets downsample algorithm - `nth`: Selects every n-th point. - `viewport`: Selects all points in a given viewport - - `viewport-xlim`: Selects all points in a given viewport, when xlim is set """ algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS)) @@ -193,23 +191,6 @@ def _process(self, element, key=None): self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") mask = slice(*self.p.x_range) element = element[mask] - elif self.p.algorithm == "viewport-xlim": - # We only want to send some of the data to the browser on the - # first pass if we have set xlim. This will render the plot and - # trigger a second pass with the x_range set. This will not work - # with the matplotlib backend because it does not update the x_range - # after the first pass, but that is not a problem because the matplotlib - # backend with the viewport algorithm does not make any sense. - # This is not very elegant. - size = element.dataset.shape[0] - mask1 = element.dataset.interface.select_mask( - element.dataset, {element.kdims[0]: slice(self.p.width // 2)}, - ) - mask2 = element.dataset.interface.select_mask( - element.dataset, {element.kdims[0]: slice(size - self.p.width // 2, size)}, - ) - element = element[mask1 | mask2] - if len(element) <= self.p.width: return element xs, ys = (element.dimension_values(i) for i in range(2)) diff --git a/holoviews/plotting/bokeh/callbacks.py b/holoviews/plotting/bokeh/callbacks.py index 8cf7639e38..a057c68e2c 100644 --- a/holoviews/plotting/bokeh/callbacks.py +++ b/holoviews/plotting/bokeh/callbacks.py @@ -645,6 +645,12 @@ class RangeXYCallback(Callback): 'y1': 'cb_obj.y1', } + def initialize(self, plot_id=None): + super().initialize(plot_id) + for stream in self.streams: + msg = self._process_msg({}) + stream.update(**msg) + def _process_msg(self, msg): if self.plot.state.x_range is not self.plot.handles['x_range']: x_range = self.plot.handles['x_range'] diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 7e3814df76..37771afb1b 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -173,9 +173,6 @@ def _update_datasource(self, source, data): """ Update datasource with data for a new frame. """ - if not self.document: - return - data = self._postprocess_data(data) empty = all(len(v) == 0 for v in data.values()) if (self.streaming and self.streaming[0].data is self.current_frame.data From aa477f467ddb4e0cd2d30c91e602925a7adcd5ef Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 24 Jan 2024 19:00:57 +0100 Subject: [PATCH 23/28] Discard change callbacks until plot is fully initialized --- holoviews/plotting/bokeh/callbacks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/holoviews/plotting/bokeh/callbacks.py b/holoviews/plotting/bokeh/callbacks.py index a057c68e2c..f6f07716d4 100644 --- a/holoviews/plotting/bokeh/callbacks.py +++ b/holoviews/plotting/bokeh/callbacks.py @@ -404,6 +404,7 @@ def set_callback(self, handle): if self.on_changes: change_handler = lambda attr, old, new: ( asyncio.create_task(self.on_change(attr, old, new)) + if self.plot.document else None ) for change in self.on_changes: if change in ['patching', 'streaming']: From a44cc549794bad8c75fd70101b2736a78c17c6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 24 Jan 2024 20:38:22 +0100 Subject: [PATCH 24/28] Update holoviews/plotting/bokeh/callbacks.py --- holoviews/plotting/bokeh/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/plotting/bokeh/callbacks.py b/holoviews/plotting/bokeh/callbacks.py index f6f07716d4..6a1bdd638c 100644 --- a/holoviews/plotting/bokeh/callbacks.py +++ b/holoviews/plotting/bokeh/callbacks.py @@ -404,7 +404,7 @@ def set_callback(self, handle): if self.on_changes: change_handler = lambda attr, old, new: ( asyncio.create_task(self.on_change(attr, old, new)) - if self.plot.document else None + if self.plot.document else None ) for change in self.on_changes: if change in ['patching', 'streaming']: From 3ddb49c74988b41fe8cb00cfae6495f38c575a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Mon, 29 Jan 2024 19:44:12 +0100 Subject: [PATCH 25/28] Update tests --- holoviews/tests/operation/test_downsample.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/holoviews/tests/operation/test_downsample.py b/holoviews/tests/operation/test_downsample.py index c3057df92c..02b491e567 100644 --- a/holoviews/tests/operation/test_downsample.py +++ b/holoviews/tests/operation/test_downsample.py @@ -9,6 +9,8 @@ except ImportError: tsdownsample = None +algorithms = _ALGORITHMS.copy() +algorithms.pop('viewport', None) # viewport return slice(len(data)) no matter the width @pytest.mark.parametrize("plottype", ["overlay", "ndoverlay"]) def test_downsample1d_multi(plottype): @@ -26,7 +28,7 @@ def test_downsample1d_multi(plottype): assert value.size == downsample1d.width -@pytest.mark.parametrize("algorithm", _ALGORITHMS.values(), ids=_ALGORITHMS) +@pytest.mark.parametrize("algorithm", algorithms.values(), ids=algorithms) def test_downsample_algorithm(algorithm, unimport): unimport("tsdownsample") x = np.arange(1000) @@ -37,14 +39,18 @@ def test_downsample_algorithm(algorithm, unimport): except NotImplementedError: pytest.skip("not testing tsdownsample algorithms") else: + if isinstance(result, slice): + result = x[result] assert result.size == width @pytest.mark.skipif(not tsdownsample, reason="tsdownsample not installed") -@pytest.mark.parametrize("algorithm", _ALGORITHMS.values(), ids=_ALGORITHMS) +@pytest.mark.parametrize("algorithm", algorithms.values(), ids=algorithms) def test_downsample_algorithm_with_tsdownsample(algorithm): x = np.arange(1000) y = np.random.rand(1000) width = 20 result = algorithm(x, y, width) + if isinstance(result, slice): + result = x[result] assert result.size == width From 2a64d0e3ac1372e1bbf80733d73a2f01ca5bf528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 30 Jan 2024 10:51:38 +0100 Subject: [PATCH 26/28] Small fixes --- holoviews/operation/downsample.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index a87aef5cb1..9c78e0e051 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -157,7 +157,7 @@ def _nth_point(x, y, n_out, **kwargs): n_samples = len(x) return slice(0, n_samples, max(1, math.ceil(n_samples / n_out))) -def _viewport(x, y, n_out): +def _viewport(x, y, n_out, **kwargs): return slice(len(x)) def _min_max(x, y, n_out, **kwargs): @@ -211,14 +211,14 @@ class downsample1d(ResampleOperation1D): algorithm = param.Selector(default='lttb', objects=list(_ALGORITHMS), doc=""" The algorithm to use for downsampling: - - `lttb`: Largest Triangle Three Buckets downsample algorithm + - `lttb`: Largest Triangle Three Buckets downsample algorithm. - `nth`: Selects every n-th point. - - `viewport`: Selects all points in a given viewport + - `viewport`: Selects all points in a given viewport. - `minmax`: Selects the min and max value in each bin (requires tsdownsampler). - `m4`: Selects the min, max, first and last value in each bin (requires tsdownsampler). - `minmax-lttb`: First selects n_out * minmax_ratio min and max values, - then further reduces these to n_out values using the - Largest Triangle Three Buckets algorithm. (requires tsdownsampler)""") + then further reduces these to n_out values using the + Largest Triangle Three Buckets algorithm (requires tsdownsampler).""") parallel = param.Boolean(default=False, doc=""" The number of threads to use (if tsdownsampler is available).""") From d6b654f628ec7b1d6acccf36a246e916b2da11bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 30 Jan 2024 10:57:46 +0100 Subject: [PATCH 27/28] Extract compute mask from _process and make it an options --- holoviews/operation/downsample.py | 36 +++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/holoviews/operation/downsample.py b/holoviews/operation/downsample.py index 9c78e0e051..9ffdd3e896 100644 --- a/holoviews/operation/downsample.py +++ b/holoviews/operation/downsample.py @@ -228,6 +228,10 @@ class downsample1d(ResampleOperation1D): values to generate with the minmax algorithm before further downsampling with LTTB.""") + neighbor_points = param.Boolean(default=None, doc=""" + Whether to add the neighbor points to the range before downsampling. + By default this is only enabled for the viewport algorithm.""") + def _process(self, element, key=None): if isinstance(element, (Overlay, NdOverlay)): _process = partial(self._process, key=key) @@ -238,15 +242,7 @@ def _process(self, element, key=None): return element.clone(elements) if self.p.x_range: - try: - mask = element.dataset.interface._select_mask_neighbor( - element.dataset, {element.kdims[0]: self.p.x_range} - ) - except NotImplementedError: - mask = slice(*self.p.x_range) - except Exception as e: - self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") - mask = slice(*self.p.x_range) + mask = self._compute_mask(element) element = element[mask] if len(element) <= self.p.width: return element @@ -263,3 +259,25 @@ def _process(self, element, key=None): kwargs['minmax_ratio'] = self.p.minmax_ratio samples = downsample(xs, ys, self.p.width, parallel=self.p.parallel, **kwargs) return element.iloc[samples] + + def _compute_mask(self, element): + """ + Computes the mask to apply to the element before downsampling. + """ + neighbor_enabled = ( + self.p.neighbor_points + if self.p.neighbor_points is not None + else self.p.algorithm == "viewport" + ) + if not neighbor_enabled: + return slice(*self.p.x_range) + try: + mask = element.dataset.interface._select_mask_neighbor( + element.dataset, {element.kdims[0]: self.p.x_range} + ) + except NotImplementedError: + mask = slice(*self.p.x_range) + except Exception as e: + self.param.warning(f"Could not apply neighbor mask to downsample1d: {e}") + mask = slice(*self.p.x_range) + return mask From 0e9ca5d8bd9cb2cd4b1bc9a7df5fb499e5c9678f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 30 Jan 2024 11:46:23 +0100 Subject: [PATCH 28/28] Ignore cuDF test --- holoviews/tests/core/data/test_cudfinterface.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/holoviews/tests/core/data/test_cudfinterface.py b/holoviews/tests/core/data/test_cudfinterface.py index cdb6c19014..87cb469190 100644 --- a/holoviews/tests/core/data/test_cudfinterface.py +++ b/holoviews/tests/core/data/test_cudfinterface.py @@ -104,3 +104,10 @@ def test_dataset_groupby_second_dim(self): def test_dataset_aggregate_string_types_size(self): raise SkipTest("cuDF does not support variance aggregation") + + def test_select_with_neighbor(self): + try: + # Not currently supported by CuDF + super().test_select_with_neighbor() + except NotImplementedError: + raise SkipTest("Not supported")