From 11c5a027057c8a167d304a650596b81aa9c95576 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 19 Oct 2017 13:12:56 +0100 Subject: [PATCH 1/9] Disable deep hashing if memoization is off --- holoviews/core/spaces.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 3f0cf59c6d..87a44221a3 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -460,6 +460,7 @@ def __init__(self, callable, **params): self._is_overlay = False self.args = None self.kwargs = None + self._memoize = self.memoize @property def argspec(self): @@ -493,12 +494,12 @@ def __call__(self, *args, **kwargs): for stream in [s for i in inputs for s in get_nested_streams(i)]: if stream not in streams: streams.append(stream) - memoize = self.memoize and not any(s.transient and s._triggering for s in streams) + memoize = self._memoize and not any(s.transient and s._triggering for s in streams) values = tuple(tuple(sorted(s.contents.items())) for s in streams) key = args + tuple(sorted(kwargs.items())) + values - hashed_key = util.deephash(key) - if memoize and hashed_key in self._memoized: + hashed_key = util.deephash(key) if self.memoize else None + if hashed_key is not None and memoize and hashed_key in self._memoized: return self._memoized[hashed_key] if self.argspec.varargs is not None: @@ -591,14 +592,14 @@ def dynamicmap_memoization(callable_obj, streams): DynamicMap). Memoization is disabled if any of the streams require it it and are currently in a triggered state. """ - memoization_state = bool(callable_obj.memoize) - callable_obj.memoize &= not any(s.transient and s._triggering for s in streams) + memoization_state = bool(callable_obj._memoize) + callable_obj._memoize &= not any(s.transient and s._triggering for s in streams) try: yield except: raise finally: - callable_obj.memoize = memoization_state + callable_obj._memoize = memoization_state From 1c89b877bbf2df9be4e796e08a6a68fbb5440e0c Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 20 Oct 2017 16:46:30 +0100 Subject: [PATCH 2/9] Minor optimization for pandas hashing --- holoviews/core/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 07de6b4e24..98e7c028f0 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -101,7 +101,7 @@ def default(self, obj): elif isinstance(obj, np.ndarray): return obj.tolist() if pd and isinstance(obj, (pd.Series, pd.DataFrame)): - return repr(sorted(list(obj.to_dict().items()))) + return obj.to_csv().encode('utf-8') elif isinstance(obj, self.string_hashable): return str(obj) elif isinstance(obj, self.repr_hashable): From 839b80c73f70fd577bcc6d4f0a7212e4ac2e678d Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 20 Oct 2017 17:12:14 +0100 Subject: [PATCH 3/9] Streams can define a hashkey property --- holoviews/core/spaces.py | 2 +- holoviews/streams.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 87a44221a3..b4d0430a35 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -495,7 +495,7 @@ def __call__(self, *args, **kwargs): if stream not in streams: streams.append(stream) memoize = self._memoize and not any(s.transient and s._triggering for s in streams) - values = tuple(tuple(sorted(s.contents.items())) for s in streams) + values = tuple(tuple(sorted(s.hashkey.items())) for s in streams) key = args + tuple(sorted(kwargs.items())) + values hashed_key = util.deephash(key) if self.memoize else None diff --git a/holoviews/streams.py b/holoviews/streams.py index 2370d46232..430f91b0ab 100644 --- a/holoviews/streams.py +++ b/holoviews/streams.py @@ -296,6 +296,15 @@ def contents(self): return {self._rename.get(k,k):v for (k,v) in filtered.items() if (self._rename.get(k,True) is not None)} + @property + def hashkey(self): + """ + The object the memoization hash is computed from. By default + returns the stream contents but can be overridden to provide + a custom hash key. + """ + return self.contents + def _set_stream_parameters(self, **kwargs): """ From 8b09a93846beca57216749be46ae7b5c3339cd68 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 20 Oct 2017 17:12:37 +0100 Subject: [PATCH 4/9] Add StreamData stream --- holoviews/streams.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/holoviews/streams.py b/holoviews/streams.py index 430f91b0ab..a7763e449d 100644 --- a/holoviews/streams.py +++ b/holoviews/streams.py @@ -4,6 +4,8 @@ server-side or in Javascript in the Jupyter notebook (client-side). """ +import uuid + import param import numpy as np from numbers import Number @@ -361,6 +363,33 @@ def transform(self): return {'counter': self.counter + 1} +class StreamData(Stream): + """ + A Stream used to pipe arbitrary data to a callback. + Unlike other streams memoization can be disabled for a + StreamData stream (and is disabled by default). + """ + + data = param.Parameter(default=None) + + def __init__(self, memoize=False, **params): + super(StreamData, self).__init__(**params) + self._memoize = memoize + + def send(self, data): + """ + A convenience method to send an event with data without + supplying a keyword. + """ + self.event(data=data) + + @property + def hashkey(self): + if self._memoize: + return self.contents + return {'hash': uuid.uuid4().hex} + + class LinkedStream(Stream): """ A LinkedStream indicates is automatically linked to plot interactions @@ -413,11 +442,11 @@ class PointerXY(LinkedStream): the plot bounds, the position values are set to None. """ - x = param.ClassSelector(class_=(Number, util.basestring), default=None, + x = param.ClassSelector(class_=(Number, util.basestring, tuple), default=None, constant=True, doc=""" Pointer position along the x-axis in data coordinates""") - y = param.ClassSelector(class_=(Number, util.basestring), default=None, + y = param.ClassSelector(class_=(Number, util.basestring, tuple), default=None, constant=True, doc=""" Pointer position along the y-axis in data coordinates""") From 3388c858645b2c9e86441e7b254b5c7fb5af4870 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 20 Oct 2017 17:29:59 +0100 Subject: [PATCH 5/9] Fixed hashing of Callable keyword args --- holoviews/core/spaces.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index b4d0430a35..e027fcab44 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -487,6 +487,7 @@ def clone(self, callable=None, **overrides): def __call__(self, *args, **kwargs): # Nothing to do for callbacks that accept no arguments + kwarg_hash = kwargs.pop('memoization_hash', []) (self.args, self.kwargs) = (args, kwargs) if not args and not kwargs: return self.callable() inputs = [i for i in self.inputs if isinstance(i, DynamicMap)] @@ -496,7 +497,7 @@ def __call__(self, *args, **kwargs): memoize = self._memoize and not any(s.transient and s._triggering for s in streams) values = tuple(tuple(sorted(s.hashkey.items())) for s in streams) - key = args + tuple(sorted(kwargs.items())) + values + key = args + tuple(sorted(kwarg_hash)) + values hashed_key = util.deephash(key) if self.memoize else None if hashed_key is not None and memoize and hashed_key in self._memoized: @@ -838,6 +839,7 @@ def _execute_callback(self, *args): # Additional validation needed to ensure kwargs don't clash kdims = [kdim.name for kdim in self.kdims] kwarg_items = [s.contents.items() for s in self.streams] + hash_items = [s.hashkey.items() for s in self.streams] flattened = [(k,v) for kws in kwarg_items for (k,v) in kws if k not in kdims] @@ -846,6 +848,7 @@ def _execute_callback(self, *args): args = () else: kwargs = dict(flattened) + kwargs['memoization_hash'] = hash_items with dynamicmap_memoization(self.callback, self.streams): retval = self.callback(*args, **kwargs) From 56d12344a6485202539c20454ab98375c8e8d47f Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 23 Oct 2017 00:49:25 +0100 Subject: [PATCH 6/9] Declared StreamData.data parameter constant --- holoviews/streams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/holoviews/streams.py b/holoviews/streams.py index a7763e449d..f0ab325489 100644 --- a/holoviews/streams.py +++ b/holoviews/streams.py @@ -370,7 +370,8 @@ class StreamData(Stream): StreamData stream (and is disabled by default). """ - data = param.Parameter(default=None) + data = param.Parameter(default=None, constant=True, doc=""" + Arbitrary data being streamed to a DynamicMap callback.""") def __init__(self, memoize=False, **params): super(StreamData, self).__init__(**params) From 9bc7efe24d43782ba47966245b32e23b465fad34 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 23 Oct 2017 00:52:21 +0100 Subject: [PATCH 7/9] Renamed Callable._memoize to Callable._stream_memoization --- holoviews/core/spaces.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index e027fcab44..e1d28e88c5 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -460,7 +460,7 @@ def __init__(self, callable, **params): self._is_overlay = False self.args = None self.kwargs = None - self._memoize = self.memoize + self._stream_memoization = self.memoize @property def argspec(self): @@ -495,7 +495,7 @@ def __call__(self, *args, **kwargs): for stream in [s for i in inputs for s in get_nested_streams(i)]: if stream not in streams: streams.append(stream) - memoize = self._memoize and not any(s.transient and s._triggering for s in streams) + memoize = self._stream_memoization and not any(s.transient and s._triggering for s in streams) values = tuple(tuple(sorted(s.hashkey.items())) for s in streams) key = args + tuple(sorted(kwarg_hash)) + values @@ -593,14 +593,14 @@ def dynamicmap_memoization(callable_obj, streams): DynamicMap). Memoization is disabled if any of the streams require it it and are currently in a triggered state. """ - memoization_state = bool(callable_obj._memoize) - callable_obj._memoize &= not any(s.transient and s._triggering for s in streams) + memoization_state = bool(callable_obj._stream_memoization) + callable_obj._stream_memoization &= not any(s.transient and s._triggering for s in streams) try: yield except: raise finally: - callable_obj._memoize = memoization_state + callable_obj._stream_memoization = memoization_state From 0757d8cf578c3e5de2b59be11d5dcb6dab616c43 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 23 Oct 2017 01:17:29 +0100 Subject: [PATCH 8/9] Added test for custom Stream.hashkey --- tests/testdynamic.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/testdynamic.py b/tests/testdynamic.py index 0fca392422..02a725a83d 100644 --- a/tests/testdynamic.py +++ b/tests/testdynamic.py @@ -1,3 +1,4 @@ +import uuid from collections import deque import time @@ -572,6 +573,35 @@ def history_callback(x, y, history=deque(maxlen=10)): self.assertEqual(xresets, 2) self.assertEqual(yresets, 2) + def test_dynamic_callable_stream_hashkey(self): + # Enable transient stream meaning memoization only happens when + # stream is inactive, should have sample for each call to + # stream.update + def history_callback(x, history=deque(maxlen=10)): + if x is not None: + history.append(x) + return Curve(list(history)) + + class NoMemoize(PointerX): + @property + def hashkey(self): return {'hash': uuid.uuid4().hex} + + x = NoMemoize() + dmap = DynamicMap(history_callback, kdims=[], streams=[x]) + + # Add stream subscriber mocking plot + x.add_subscriber(lambda **kwargs: dmap[()]) + + for i in range(2): + x.event(x=1) + self.assertEqual(dmap[()], Curve([1, 1, 1])) + + for i in range(2): + x.event(x=2) + + self.assertEqual(dmap[()], Curve([1, 1, 1, 2, 2, 2])) + + class TestPeriodicStreamUpdate(ComparisonTestCase): From b63a3841cf3e64b2a27b3deb8a8bbb7edcdf825e Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 23 Oct 2017 01:59:11 +0100 Subject: [PATCH 9/9] Ensure hash_items are sorted --- holoviews/core/spaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index e1d28e88c5..211ec2dbde 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -839,7 +839,7 @@ def _execute_callback(self, *args): # Additional validation needed to ensure kwargs don't clash kdims = [kdim.name for kdim in self.kdims] kwarg_items = [s.contents.items() for s in self.streams] - hash_items = [s.hashkey.items() for s in self.streams] + hash_items = tuple(tuple(sorted(s.hashkey.items())) for s in self.streams) flattened = [(k,v) for kws in kwarg_items for (k,v) in kws if k not in kdims]