Rolling window with as_strided

doc/computation.rst

@@ -158,20 +158,37 @@ Aggregation and summary methods can be applied directly to the ``Rolling`` objec
     r.mean()
     r.reduce(np.std)
 
-Note that rolling window aggregations are much faster (both asymptotically and
-because they avoid a loop in Python) when bottleneck_ is installed. Otherwise,
-we fall back to a slower, pure Python implementation.
+Note that rolling window aggregations are faster when bottleneck_ is installed.
 
 .. _bottleneck: https://github.com/kwgoodman/bottleneck/
 
-Finally, we can manually iterate through ``Rolling`` objects:
+We can also manually iterate through ``Rolling`` objects:
 
 .. ipython:: python
 
    @verbatim
    for label, arr_window in r:
       # arr_window is a view of x
 
+Finally, the rolling object has ``to_dataarray`` method
+(``to_dataset`` method for Rolling objects from Dataset), which gives a
+view of the original ``DataArray`` with the windowed dimension attached to
+the last position.
+You can use this for more advanced rolling operations, such as strided rolling,
+windowed rolling, convolution, short-time FFT, etc.
+
+.. ipython:: python
+
+    rolling_da = r.to_dataarray('window_dim')
+    rolling_da
+    # rolling mean with 2-point stride
+    rolling_da.isel(y=slice(None, None, 2)).mean('window_dim')
+
+Note that although the ``DataArray`` obtained by
+``r.to_dataarray('window_dim')`` has an additional dimension,
+it does not consume too much memory as it is just a view of
+the original array.
+
 .. _compute.broadcasting:
 
 Broadcasting by dimension name

doc/whats-new.rst

@@ -27,7 +27,13 @@ Documentation
 
 Enhancements
 ~~~~~~~~~~~~
-- reduce methods such as :py:func:`DataArray.sum()` now accepts ``dtype``
+- Improve :py:func:`~xarray.DataArray.rooling` logic for speed up.
+  :py:func:`~xarray.DataArrayRolling` object now support ``to_dataarray``
+  method that returns a view of the DataArray object with the rolling-window
+  dimension added to the last position. This enables more flexible operation,
+  such as strided rolling, windowed rolling, ND-rolling, and convolution.
+  (:issue:`1831`, :issue:`1142`, :issue:`819`)
+- reduce methods such as :py:func:`DataArray.sum()` now accept ``dtype``
   arguments. (:issue:`1838`)
   By `Keisuke Fujii <https://github.com/fujiisoup>`_.
 - Added nodatavals attribute to DataArray when using :py:func:`~xarray.open_rasterio`. (:issue:`1736`).
@@ -68,6 +74,10 @@ Enhancements
 
 Bug fixes
 ~~~~~~~~~
+- Rolling aggregation with ``center=True`` option now gives the same result
+  with pandas including the last element (:issue:`1046`).
+  By `Keisuke Fujii <https://github.com/fujiisoup>`_.
+
 - Added warning in api.py of a netCDF4 bug that occurs when
   the filepath has 88 characters (:issue:`1745`).
   By `Liam Brannigan <https://github.com/braaannigan>` _.

xarray/core/duck_array_ops.py

@@ -16,6 +16,7 @@
 import pandas as pd
 
 from . import npcompat
+from . import nputils
 from . import dtypes
 from .pycompat import dask_array_type
 from .nputils import nanfirst, nanlast
@@ -265,3 +266,24 @@ def last(values, axis, skipna=None):
         _fail_on_dask_array_input_skipna(values)
         return nanlast(values, axis)
     return take(values, -1, axis=axis)
+
+
+def rolling_window(array, axis, window):
+    """
+    Make an ndarray with a rolling window of axis-th dimension.
+    The rolling dimension will be placed at the last dimension.
+    """
+    if isinstance(array, dask_array_type):
+        if window < 1:
+            raise ValueError(
+                "`window` must be at least 1. Given : {}".format(window))
+        if window > array.shape[axis]:
+            raise ValueError("`window` is too long. Given : {}".format(window))
+
+        axis = nputils._validate_axis(array, axis)
+        size = array.shape[axis] - window + 1
+        arrays = [array[(slice(None), ) * axis + (slice(w, size + w), )]
+                  for w in range(window)]
+        return da.stack(arrays, axis=-1)
+    else:  # np.ndarray
+        return nputils.rolling_window(array, axis, window)

xarray/core/npcompat.py

@@ -2,6 +2,18 @@
 from __future__ import division
 from __future__ import print_function
 import numpy as np
+from distutils.version import LooseVersion
+
+
+if LooseVersion(np.__version__) < LooseVersion('1.12'):
+    def as_strided(x, shape=None, strides=None, subok=False, writeable=True):
+        array = np.lib.stride_tricks.as_strided(x, shape, strides, subok)
+        array.setflags(write=writeable)
+        return array
+
+else:
+    as_strided = np.lib.stride_tricks.as_strided
+
 
 try:
     from numpy import nancumsum, nancumprod, flip

xarray/core/nputils.py

@@ -4,6 +4,7 @@
 import numpy as np
 import pandas as pd
 import warnings
+from . import npcompat
 
 
 def _validate_axis(data, axis):
@@ -133,3 +134,52 @@ def __setitem__(self, key, value):
         mixed_positions, vindex_positions = _advanced_indexer_subspaces(key)
         self._array[key] = np.moveaxis(value, vindex_positions,
                                        mixed_positions)
+
+
+def rolling_window(a, axis, window):
+    """
+    Make an ndarray with a rolling window along axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Array to add rolling window to
+    axis: int
+        axis position along which rolling window will be applied.
+    window : int
+        Size of rolling window
+
+    Returns
+    -------
+    Array that is a view of the original array with a added dimension
+    of size w.
+
+    Examples
+    --------
+    >>> x=np.arange(10).reshape((2,5))
+    >>> np.rolling_window(x, 3, axis=-1)
+    array([[[0, 1, 2], [1, 2, 3], [2, 3, 4]],
+           [[5, 6, 7], [6, 7, 8], [7, 8, 9]]])
+
+    Calculate rolling mean of last dimension:
+    >>> np.mean(np.rolling_window(x, 3, axis=-1), -1)
+    array([[ 1.,  2.,  3.],
+           [ 6.,  7.,  8.]])
+
+    This function is taken from https://github.com/numpy/numpy/pull/31
+    but slightly modified to accept axis option.
+    """
+    axis = _validate_axis(a, axis)
+    a = np.swapaxes(a, axis, -1)
+
+    if window < 1:
+        raise ValueError(
+            "`window` must be at least 1. Given : {}".format(window))
+    if window > a.shape[-1]:
+        raise ValueError("`window` is too long. Given : {}".format(window))
+
+    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
+    strides = a.strides + (a.strides[-1],)
+    rolling = npcompat.as_strided(a, shape=shape, strides=strides,
+                                  writeable=False)
+    return np.swapaxes(rolling, -2, axis)

xarray/core/ops.py

@@ -227,20 +227,8 @@ def func(self, *args, **kwargs):
 
 def rolling_count(rolling):
 
-    not_null = rolling.obj.notnull()
-    instance_attr_dict = {'center': rolling.center,
-                          'min_periods': rolling.min_periods,
-                          rolling.dim: rolling.window}
-    rolling_count = not_null.rolling(**instance_attr_dict).sum()
-
-    if rolling.min_periods is None:
-        return rolling_count
-
-    # otherwise we need to filter out points where there aren't enough periods
-    # but not_null is False, and so the NaNs don't flow through
-    # array with points where there are enough values given min_periods
-    enough_periods = rolling_count >= rolling.min_periods
-
+    rolling_count = rolling._counts()
+    enough_periods = rolling_count > rolling._min_periods - 0.5
     return rolling_count.where(enough_periods)