Added support for cftime types (#2728)

holoviews/core/ndmapping.py

@@ -650,7 +650,7 @@ def __getitem__(self, indexslice):
             return self.data[()]
         elif indexslice in [Ellipsis, ()]:
             return self
-        elif Ellipsis in wrap_tuple(indexslice):
+        elif any(Ellipsis is sl for sl in wrap_tuple(indexslice)):
             indexslice = process_ellipses(self, indexslice)
 
         map_slice, data_slice = self._split_index(indexslice)

holoviews/core/util.py

@@ -92,6 +92,14 @@ def __cmp__(self, other):
 except ImportError:
     pd = None
 
+try:
+    import cftime
+    cftime_types = (cftime.datetime,)
+    datetime_types += cftime_types
+except:
+    cftime_types = ()
+_STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian'])
+
 
 class VersionError(Exception):
     "Raised when there is a library version mismatch."
@@ -1805,26 +1813,28 @@ def dt_to_int(value, time_unit='us'):
         if isinstance(value, pd.Period):
             value = value.to_timestamp()
         if isinstance(value, pd.Timestamp):
-            value = value.to_pydatetime()
-        value = np.datetime64(value)
+            try:
+                value = value.to_datetime64()
+            except:
+                value = np.datetime64(value.to_pydatetime())
+    elif isinstance(value, cftime_types):
+        return cftime_to_timestamp(value, time_unit)
 
+    # Handle datetime64 separately
     if isinstance(value, np.datetime64):
-        value = np.datetime64(value, 'ns')
-        if time_unit == 'ns':
-            tscale = 1
-        else:
-            tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns')) * 1000.
-    elif time_unit == 'ns':
-        tscale = 1000.
+        try:
+            value = np.datetime64(value, 'ns')
+            tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns'))
+            return value.tolist()/tscale
+        except:
+            # If it can't handle ns precision fall back to datetime
+            value = value.tolist()
+
+    if time_unit == 'ns':
+        tscale = 1e9
     else:
         tscale = 1./np.timedelta64(1, time_unit).tolist().total_seconds()
 
-    if isinstance(value, np.datetime64):
-        value = value.tolist()
-    if isinstance(value, (int, long)):
-        # Handle special case of nanosecond precision which cannot be
-        # represented by python datetime
-        return value * 10**-(np.log10(tscale)-3)
     try:
         # Handle python3
         return int(value.timestamp() * tscale)
@@ -1833,6 +1843,30 @@ def dt_to_int(value, time_unit='us'):
         return (time.mktime(value.timetuple()) + value.microsecond / 1e6) * tscale
 
 
+def cftime_to_timestamp(date, time_unit='us'):
+    """Converts cftime to timestamp since epoch in milliseconds
+
+    Non-standard calendars (e.g. Julian or no leap calendars)
+    are converted to standard Gregorian calendar. This can cause
+    extra space to be added for dates that don't exist in the original
+    calendar. In order to handle these dates correctly a custom bokeh
+    model with support for other calendars would have to be defined.
+
+    Args:
+        date: cftime datetime object (or array)
+
+    Returns:
+        Milliseconds since 1970-01-01 00:00:00
+    """
+    import cftime
+    utime = cftime.utime('microseconds since 1970-01-01')
+    if time_unit == 'us':
+        tscale = 1
+    else:
+        tscale = (np.timedelta64(1, 'us')/np.timedelta64(1, time_unit))
+    return utime.date2num(date)*tscale
+
+
 def search_indices(values, source):
     """
     Given a set of values returns the indices of each of those values

holoviews/operation/datashader.py

@@ -23,7 +23,9 @@
                     CompositeOverlay, Dataset, Overlay)
 from ..core.data import PandasInterface, XArrayInterface
 from ..core.sheetcoords import BoundingBox
-from ..core.util import LooseVersion, get_param_values, basestring, datetime_types, dt_to_int
+from ..core.util import (
+    LooseVersion, basestring, cftime_types, cftime_to_timestamp,
+    datetime_types, dt_to_int, get_param_values)
 from ..element import (Image, Path, Curve, RGB, Graph, TriMesh, QuadMesh, Contours)
 from ..streams import RangeXY, PlotSize
 
@@ -321,11 +323,18 @@ def get_agg_data(cls, obj, category=None):
         if category and df[category].dtype.name != 'category':
             df[category] = df[category].astype('category')
 
-        if any(df[d.name].dtype.kind == 'M' for d in (x, y)):
+        if any(df[d.name].dtype.kind == 'M' or isinstance(df[d.name].values[0], cftime_types)
+               for d in (x, y)):
             df = df.copy()
         for d in (x, y):
-            if df[d.name].dtype.kind == 'M':
-                df[d.name] = df[d.name].astype('datetime64[ns]').astype('int64') * 1000.
+            vals = df[d.name].values
+            if len(vals) and isinstance(vals[0], cftime_types):
+                vals = cftime_to_timestamp(vals, 'ns')
+            elif df[d.name].dtype.kind == 'M':
+                vals = vals.astype('datetime64[ns]')
+            else:
+                continue
+            df[d.name] = vals.astype('int64')
         return x, y, Dataset(df, kdims=kdims, vdims=vdims), glyph
 
 
@@ -344,9 +353,9 @@ def _aggregate_ndoverlay(self, element, agg_fn):
         info = self._get_sampling(element, x, y)
         (x_range, y_range), (xs, ys), (width, height), (xtype, ytype) = info
         if xtype == 'datetime':
-            x_range = tuple((np.array(x_range)/10e5).astype('datetime64[us]'))
+            x_range = tuple((np.array(x_range)/1e3).astype('datetime64[us]'))
         if ytype == 'datetime':
-            y_range = tuple((np.array(y_range)/10e5).astype('datetime64[us]'))
+            y_range = tuple((np.array(y_range)/1e3).astype('datetime64[us]'))
         agg_params = dict({k: v for k, v in dict(self.get_param_values(), **self.p).items()
                            if k in aggregate.params()},
                           x_range=x_range, y_range=y_range)
@@ -433,11 +442,11 @@ def _process(self, element, key=None):
 
         (x0, x1), (y0, y1) = x_range, y_range
         if xtype == 'datetime':
-            x0, x1 = (np.array([x0, x1])/10e5).astype('datetime64[us]')
-            xs = (xs/10e5).astype('datetime64[us]')
+            x0, x1 = (np.array([x0, x1])/1e3).astype('datetime64[us]')
+            xs = (xs/1e3).astype('datetime64[us]')
         if ytype == 'datetime':
-            y0, y1 = (np.array([y0, y1])/10e5).astype('datetime64[us]')
-            ys = (ys/10e5).astype('datetime64[us]')
+            y0, y1 = (np.array([y0, y1])/1e3).astype('datetime64[us]')
+            ys = (ys/1e3).astype('datetime64[us]')
         bounds = (x0, y0, x1, y1)
         params = dict(get_param_values(element), kdims=[x, y],
                       datatype=['xarray'], bounds=bounds)
@@ -483,9 +492,9 @@ def _process(self, element, key=None):
         if 'x_axis' in agg.coords and 'y_axis' in agg.coords:
             agg = agg.rename({'x_axis': x, 'y_axis': y})
         if xtype == 'datetime':
-            agg[x.name] = (agg[x.name]/10e5).astype('datetime64[us]')
+            agg[x.name] = (agg[x.name]/1e3).astype('datetime64[us]')
         if ytype == 'datetime':
-            agg[y.name] = (agg[y.name]/10e5).astype('datetime64[us]')
+            agg[y.name] = (agg[y.name]/1e3).astype('datetime64[us]')
 
         if agg.ndim == 2:
             # Replacing x and y coordinates to avoid numerical precision issues
@@ -606,11 +615,11 @@ def _process(self, element, key=None):
 
         # Compute bounds (converting datetimes)
         if xtype == 'datetime':
-            xstart, xend = (np.array([xstart, xend])/10e5).astype('datetime64[us]')
-            xs = (xs/10e5).astype('datetime64[us]')
+            xstart, xend = (np.array([xstart, xend])/1e3).astype('datetime64[us]')
+            xs = (xs/1e3).astype('datetime64[us]')
         if ytype == 'datetime':
-            ystart, yend = (np.array([ystart, yend])/10e5).astype('datetime64[us]')
-            ys = (ys/10e5).astype('datetime64[us]')
+            ystart, yend = (np.array([ystart, yend])/1e3).astype('datetime64[us]')
+            ys = (ys/1e3).astype('datetime64[us]')
         bbox = BoundingBox(points=[(xstart, ystart), (xend, yend)])
 
         params = dict(bounds=bbox)
@@ -632,9 +641,9 @@ def _process(self, element, key=None):
 
             # Convert datetime coordinates
             if xtype == "datetime":
-                rarray[x.name] = (rarray[x.name]/10e5).astype('datetime64[us]')
+                rarray[x.name] = (rarray[x.name]/1e3).astype('datetime64[us]')
             if ytype == "datetime":
-                rarray[y.name] = (rarray[y.name]/10e5).astype('datetime64[us]')
+                rarray[y.name] = (rarray[y.name]/1e3).astype('datetime64[us]')
             regridded[vd] = rarray
         regridded = xr.Dataset(regridded)
 
@@ -957,7 +966,7 @@ def _process(self, element, key=None):
 
         for d in kdims:
             if array[d.name].dtype.kind == 'M':
-                array[d.name] = array[d.name].astype('datetime64[ns]').astype('int64') * 10e-4
+                array[d.name] = array[d.name].astype('datetime64[us]').astype('int64')
 
         with warnings.catch_warnings():
             warnings.filterwarnings('ignore', r'invalid value encountered in true_divide')

holoviews/operation/element.py

@@ -589,9 +589,9 @@ def _process(self, element, key=None):
         if data.dtype.kind == 'M' or (data.dtype.kind == 'O' and isinstance(data[0], datetime_types)):
             start, end = dt_to_int(start, 'ns'), dt_to_int(end, 'ns')
             datetimes = True
-            data = data.astype('datetime64[ns]').astype('int64') * 1000.
+            data = data.astype('datetime64[ns]').astype('int64')
             if bins is not None:
-                bins = bins.astype('datetime64[ns]').astype('int64') * 1000.
+                bins = bins.astype('datetime64[ns]').astype('int64')
             else:
                 hist_range = start, end
 
@@ -622,7 +622,7 @@ def _process(self, element, key=None):
             hist = np.zeros(self.p.num_bins)
         hist[np.isnan(hist)] = 0
         if datetimes:
-            edges = (edges/10e5).astype('datetime64[us]')
+            edges = (edges/1e3).astype('datetime64[us]')
 
         params = {}
         if self.p.weight_dimension:

holoviews/plotting/bokeh/element.py

@@ -40,7 +40,7 @@
 from .util import (
     bokeh_version, decode_bytes, get_tab_title, glyph_order,
     py2js_tickformatter, recursive_model_update, theme_attr_json,
-    cds_column_replace, hold_policy, match_dim_specs
+    cds_column_replace, hold_policy, match_dim_specs, date_to_integer
 )
 
 
@@ -301,7 +301,7 @@ def _axes_props(self, plots, subplots, element, ranges):
                     xtype = el.nodes.get_dimension_type(xdims[0])
                 else:
                     xtype = el.get_dimension_type(xdims[0])
-                if ((xtype is np.object_ and type(l) in util.datetime_types) or
+                if ((xtype is np.object_ and issubclass(type(l), util.datetime_types)) or
                     xtype in util.datetime_types):
                     x_axis_type = 'datetime'
 
@@ -315,7 +315,7 @@ def _axes_props(self, plots, subplots, element, ranges):
                     ytype = el.nodes.get_dimension_type(ydims[0])
                 else:
                     ytype = el.get_dimension_type(ydims[0])
-                if ((ytype is np.object_ and type(b) in util.datetime_types)
+                if ((ytype is np.object_ and issubclass(type(b), util.datetime_types))
                     or ytype in util.datetime_types):
                     y_axis_type = 'datetime'
 
@@ -606,7 +606,9 @@ def _update_ranges(self, element, ranges):
 
     def _update_range(self, axis_range, low, high, factors, invert, shared, log, streaming=False):
         if isinstance(axis_range, (Range1d, DataRange1d)) and self.apply_ranges:
-            if (low == high and low is not None):
+            if isinstance(low, util.cftime_types):
+                pass
+            elif (low == high and low is not None):
                 if isinstance(low, util.datetime_types):
                     offset = np.timedelta64(500, 'ms')
                     low -= offset
@@ -634,6 +636,8 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str
                 if reset_supported:
                     updates['reset_end'] = updates['end']
             for k, (old, new) in updates.items():
+                if isinstance(new, util.cftime_types):
+                    new = date_to_integer(new)
                 axis_range.update(**{k:new})
                 if streaming and not k.startswith('reset_'):
                     axis_range.trigger(k, old, new)

holoviews/plotting/bokeh/plot.py

@@ -12,20 +12,27 @@
 from bokeh.models.widgets import Panel, Tabs
 from bokeh.plotting.helpers import _known_tools as known_tools
 
-from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout,
-                     Empty, GridSpace, HoloMap, Element, DynamicMap)
+from ...core import (
+    OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty,
+    GridSpace, HoloMap, Element, DynamicMap
+)
 from ...core.options import SkipRendering
-from ...core.util import (basestring, wrap_tuple, unique_iterator,
-                          get_method_owner, wrap_tuple_streams)
+from ...core.util import (
+    basestring, cftime_to_timestamp, cftime_types, get_method_owner,
+    unique_iterator, wrap_tuple, wrap_tuple_streams, _STANDARD_CALENDARS)
 from ...streams import Stream
 from ..links import Link
-from ..plot import (DimensionedPlot, GenericCompositePlot, GenericLayoutPlot,
-                    GenericElementPlot, GenericOverlayPlot)
+from ..plot import (
+    DimensionedPlot, GenericCompositePlot, GenericLayoutPlot,
+    GenericElementPlot, GenericOverlayPlot
+)
 from ..util import attach_streams, displayable, collate
 from .callbacks import LinkCallback
-from .util import (layout_padding, pad_plots, filter_toolboxes, make_axis,
-                   update_shared_sources, empty_plot, decode_bytes,
-                   theme_attr_json, cds_column_replace)
+from .util import (
+    layout_padding, pad_plots, filter_toolboxes, make_axis,
+    update_shared_sources, empty_plot, decode_bytes, theme_attr_json,
+    cds_column_replace
+)
 
 TOOLS = {name: tool if isinstance(tool, basestring) else type(tool())
          for name, tool in known_tools.items()}
@@ -226,18 +233,41 @@ def _init_datasource(self, data):
         """
         Initializes a data source to be passed into the bokeh glyph.
         """
-        data = {k: decode_bytes(vs) for k, vs in data.items()}
+        data = self._postprocess_data(data)
         return ColumnDataSource(data=data)
 
 
+    def _postprocess_data(self, data):
+        """
+        Applies necessary type transformation to the data before
+        it is set on a ColumnDataSource.
+        """
+        new_data = {}
+        for k, values in data.items():
+            values = decode_bytes(values) # Bytes need decoding to strings
+
+            # Certain datetime types need to be converted
+            if len(values) and isinstance(values[0], cftime_types):
+                if any(v.calendar not in _STANDARD_CALENDARS for v in values):
+                    self.param.warning(
+                        'Converting cftime.datetime from a non-standard '
+                        'calendar (%s) to a standard calendar for plotting. '
+                        'This may lead to subtle errors in formatting '
+                        'dates, for accurate tick formatting switch to '
+                        'the matplotlib backend.' % values[0].calendar)
+                values = cftime_to_timestamp(values, 'ms')
+            new_data[k] = values
+        return new_data
+
+
     def _update_datasource(self, source, data):
         """
         Update datasource with data for a new frame.
         """
         if not self.document:
             return
 
-        data = {k: decode_bytes(vs) for k, vs in data.items()}
+        data = self._postprocess_data(data)
         empty = all(len(v) == 0 for v in data.values())
         if (self.streaming and self.streaming[0].data is self.current_frame.data
             and self._stream_data and not empty):