#200: added support for xarray datasets

.circleci/config.yml

@@ -66,7 +66,7 @@ python: &python
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0
-      VERSION: 1.8.15
+      VERSION: 1.8.16
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
     steps:
       - attach_workspace:

CHANGES.md

@@ -1,5 +1,8 @@
 ## Changelog
 
+### 1.8.16 (2020-6-7)
+ * [#200](https://github.com/man-group/dtale/issues/200): support for xarray
+
 ### 1.8.15 (2020-5-31)
  * [#202](https://github.com/man-group/dtale/issues/202): maximum recursion errors when using Pyzo IDE
 

README.md

@@ -47,7 +47,7 @@ D-Tale was the product of a SAS to Python conversion.  What was originally a per
   - [Header](#header)
   - [Editing Cells](#editing-cells)
   - [Main Menu Functions](#main-menu-functions)
-    - [Describe](#describe), [Outlier Detection](#outlier-detection), [Custom Filter](#custom-filter), [Building Columns](#building-columns), [Summarize Data](#summarize-data), [Charts](#charts), [Coverage (Deprecated)](#coverage-deprecated), [Correlations](#correlations), [Heat Map](#heat-map), [Highlight Dtypes](#highlight-dtypes), [Highlight Missing](#highlight-missing), [Highlight Outliers](#highlight-outliers), [Highlight Range](#highlight-range), [Instances](#instances), [Code Exports](#code-exports), [About](#about), [Resize](#resize), [Shutdown](#shutdown)
+    - [XArray Operations](#xarray-operations), [Describe](#describe), [Outlier Detection](#outlier-detection), [Custom Filter](#custom-filter), [Building Columns](#building-columns), [Summarize Data](#summarize-data), [Charts](#charts), [Coverage (Deprecated)](#coverage-deprecated), [Correlations](#correlations), [Heat Map](#heat-map), [Highlight Dtypes](#highlight-dtypes), [Highlight Missing](#highlight-missing), [Highlight Outliers](#highlight-outliers), [Highlight Range](#highlight-range), [Instances](#instances), [Code Exports](#code-exports), [About](#about), [Resize](#resize), [Shutdown](#shutdown)
   - [Column Menu Functions](#column-menu-functions)
     - [Filtering](#filtering), [Moving Columns](#moving-columns), [Hiding Columns](#hiding-columns), [Delete](#delete), [Rename](#rename), [Replacements](#replacements), [Lock](#lock), [Unlock](#unlock), [Sorting](#sorting), [Formats](#formats), [Column Analysis](#column-analysis)
   - [Menu Functions Depending on Browser Dimensions](#menu-functions-depending-on-browser-dimensions)
@@ -406,6 +406,16 @@ Here's a quick demo:
 
 ### Main Menu Functions
 
+#### XArray Operations
+
+* **Convert To XArray**: If you have are currently viewing a pandas dataframe in D-Tale you will be given this option to convert your data to an `xarray.Dataset`.  It is as simple as selecting one or many columns as an index and then your dataframe will be converted to a dataset (`df.set_index([...]).to_xarray()`) which makes toggling between indexes slices much easier.
+
+![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/xarray_indexes.png)
+
+* **XArray Dimensions**: If you are currently viewing data associated with an `xarray.Dataset` you will be given the ability to toggle which dimension coordinates you're viewing by clicking this button.  You can select values for all, some or none (all data - no filter) of your coordinates and the data displayed in your grid will match your selections.  Under the hood the code being executed is as follows: `ds.sel(dim1=coord1,...).to_dataframe()`
+
+![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/xarray_dimensions.png)
+
 #### Describe
 View all the columns & their data types as well as individual details of each column
 

docker/dtale.env

@@ -1,2 +1,2 @@
-VERSION=1.8.15
+VERSION=1.8.16
 TZ=America/New_York

docs/source/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'1.8.15'
+version = u'1.8.16'
 # The full version, including alpha/beta/rc tags.
-release = u'1.8.15'
+release = u'1.8.16'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/requirements.txt

@@ -11,4 +11,5 @@ pandas
 future
 Flask
 scipy
-scikit-learn
+scikit-learn
+xarray

dtale/cli/clickutils.py

@@ -35,10 +35,6 @@ def setup_logging(logfile, log_level, verbose=False):
     else:
         log_level = LOG_LEVELS['info']
 
-    # for pkg in ['_plotly_utils', 'asyncio', 'concurrent', 'matplotlib', 'parso', 'past', 'prompt_toolkit', 'requests',
-    #             'tornado', 'urllib3']:
-    #     logging.getLogger(pkg).propagate = True
-
     logging.getLogger().handlers = []
 
     fmt = "%(asctime)s - %(levelname)-8s - %(message)s"

dtale/global_state.py

@@ -2,6 +2,8 @@
 from six.moves.collections_abc import MutableMapping
 
 DATA = {}
+DATASETS = {}
+DATASET_DIM = {}
 DTYPES = {}
 SETTINGS = {}
 METADATA = {}
@@ -17,6 +19,22 @@ def get_data(data_id=None):
     return DATA.get(data_id)
 
 
+def get_dataset(data_id=None):
+    global DATASETS
+
+    if data_id is None:
+        return _as_dict(DATASETS)
+    return DATASETS.get(data_id)
+
+
+def get_dataset_dim(data_id=None):
+    global DATASET_DIM
+
+    if data_id is None:
+        return _as_dict(DATASET_DIM)
+    return DATASET_DIM.get(data_id)
+
+
 def get_dtypes(data_id=None):
     global DTYPES
 
@@ -63,6 +81,18 @@ def set_data(data_id, val):
     DATA[data_id] = val
 
 
+def set_dataset(data_id, val):
+    global DATASETS
+
+    DATASETS[data_id] = val
+
+
+def set_dataset_dim(data_id, val):
+    global DATASET_DIM
+
+    DATASET_DIM[data_id] = val
+
+
 def set_dtypes(data_id, val):
     global DTYPES
 
@@ -100,17 +130,19 @@ def cleanup(data_id=None):
     :param port: integer string for a D-Tale process's port
     :type port: str
     """
-    global DATA, DTYPES, SETTINGS, METADATA, CONTEXT_VARIABLES, HISTORY
+    global DATA, DATASETS, DATASET_DIM, DTYPES, SETTINGS, METADATA, CONTEXT_VARIABLES, HISTORY
 
     if data_id is None:
         DATA.clear()
+        DATASETS.clear()
+        DATASET_DIM.clear()
         SETTINGS.clear()
         DTYPES.clear()
         METADATA.clear()
         CONTEXT_VARIABLES.clear()
         HISTORY.clear()
     else:
-        for store in [DATA, DTYPES, SETTINGS, METADATA, CONTEXT_VARIABLES, HISTORY]:
+        for store in [DATA, DATASETS, DATASET_DIM, DTYPES, SETTINGS, METADATA, CONTEXT_VARIABLES, HISTORY]:
             if data_id in store:
                 del store[data_id]
 

dtale/static/css/main.css

@@ -4564,6 +4564,14 @@ div.filter-modal > div.modal-lg {
   min-width: 1100px;
 }
 
+div.xarray-dimensions-modal > div.modal-lg {
+  width: 600px;
+}
+
+div.xarray-indexes-modal > div.modal-lg{
+  width: 500px;
+}
+
 @media (min-width: 992px) {
   .modal-lg {
     max-width: 800px;

dtale/templates/dtale/base.html

@@ -30,6 +30,8 @@
         </span>
         {% endif %}
         <input type="hidden" id="data_id" value="{{data_id}}" />
+        <input type="hidden" id="xarray" value="{{xarray}}" />
+        <input type="hidden" id="xarray_dim" value="{{xarray_dim}}" />
         <input type="hidden" id="settings" value="{{settings}}" />
         <input type="hidden" id="version" value="{{version}}" />
         <input type="hidden" id="hide_shutdown" value="{{config.HIDE_SHUTDOWN}}" />

dtale/utils.py

@@ -729,18 +729,33 @@ def build_code_export(data_id, imports='import pandas as pd\n\n', query=None):
     settings = global_state.get_settings(data_id) or {}
     ctxt_vars = global_state.get_context_variables(data_id)
 
-    startup_code = settings.get('startup_code')
-    startup_code = '# Data Re-shaping\n{}\n\n'.format(startup_code) if startup_code else ''
+    startup_code = settings.get('startup_code') or ''
+    xarray_setup = ''
+    if data_id in global_state.DATASETS:
+        xarray_dims = global_state.get_dataset_dim(data_id)
+        if len(xarray_dims):
+            xarray_setup = (
+                'df = ds.sel({selectors}).to_dataframe()\n'
+                "df = df.reset_index().drop('index', axis=1, errors='ignore')\n"
+                'df = df.set_index(list(ds.dims.keys()))\n'
+            ).format(selectors=', '.join("{}='{}'".format(k, v) for k, v in xarray_dims.items()))
+        else:
+            xarray_setup = (
+                'df = ds.to_dataframe()\n'
+                "df = df.reset_index().drop('index', axis=1, errors='ignore')\n"
+                'df = df.set_index(list(ds.dims.keys()))\n'
+            )
     startup_str = (
         "# DISCLAIMER: 'df' refers to the data you passed in when calling 'dtale.show'\n\n"
         '{imports}'
+        '{xarray_setup}'
         '{startup}'
         'if isinstance(df, (pd.DatetimeIndex, pd.MultiIndex)):\n'
         '\tdf = df.to_frame(index=False)\n\n'
         '# remove any pre-existing indices for ease of use in the D-Tale code, but this is not required\n'
         "df = df.reset_index().drop('index', axis=1, errors='ignore')\n"
         'df.columns = [str(c) for c in df.columns]  # update columns to strings in case they are numbers\n'
-    ).format(imports=imports, startup=startup_code)
+    ).format(imports=imports, xarray_setup=xarray_setup, startup=startup_code)
     final_history = [startup_str] + history
     final_query = query
     if final_query is None: