Script consistency

datacube/api/_api.py

@@ -22,7 +22,7 @@
 import numpy
 
 from ..utils import geometry
-from .core import Datacube, Group, get_bounds, datatset_type_to_row
+from .core import Datacube, Group, get_bounds, dataset_type_to_row
 from .query import DescriptorQuery
 
 _LOG = logging.getLogger(__name__)
@@ -411,7 +411,7 @@ def list_products(self):
 
         :return: list of dictionaries describing the products
         """
-        return [datatset_type_to_row(dataset_type) for dataset_type in self.datacube.index.products.get_all()]
+        return [dataset_type_to_row(dataset_type) for dataset_type in self.datacube.index.products.get_all()]
 
     def list_variables(self):
         """

datacube/api/core.py

@@ -103,7 +103,7 @@ def list_products(self, show_archived=False, with_pandas=True):
         :param with_pandas: return the list as a Pandas DataFrame, otherwise as a list of dict.
         :rtype: pandas.DataFrame or list(dict)
         """
-        rows = [datatset_type_to_row(dataset_type) for dataset_type in self.index.products.get_all()]
+        rows = [dataset_type_to_row(dataset_type) for dataset_type in self.index.products.get_all()]
         if not with_pandas:
             return rows
 
@@ -661,7 +661,7 @@ def make_resampled_measurement(measurement):
     return measurements
 
 
-def datatset_type_to_row(dt):
+def dataset_type_to_row(dt):
     row = {
         'id': dt.id,
         'name': dt.name,
@@ -670,7 +670,7 @@ def datatset_type_to_row(dt):
     row.update(dt.fields)
     if dt.grid_spec is not None:
         row.update({
-            'crs': dt.grid_spec.crs,
+            'crs': str(dt.grid_spec.crs),
             'spatial_dimensions': dt.grid_spec.dimensions,
             'tile_size': dt.grid_spec.tile_size,
             'resolution': dt.grid_spec.resolution,

datacube/scripts/dataset.py

@@ -23,6 +23,7 @@
 from datacube.ui.click import cli
 from datacube.ui.common import get_metadata_path
 from datacube.utils import read_documents, changes, InvalidDocException
+from datacube.utils.serialise import SafeDatacubeDumper
 
 try:
     from typing import Iterable
@@ -279,7 +280,6 @@ def build_dataset_info(index, dataset, show_sources=False, show_derived=False, d
 
     return info
 
-
 def _write_csv(infos):
     writer = csv.DictWriter(sys.stdout, ['id', 'status', 'product', 'location'], extrasaction='ignore')
     writer.writeheader()
@@ -291,7 +291,6 @@ def add_first_location(row):
 
     writer.writerows(add_first_location(row) for row in infos)
 
-
 def _write_yaml(infos):
     """
     Dump yaml data with support for OrderedDicts.
@@ -301,48 +300,13 @@ def _write_yaml(infos):
     (Ordered dicts are output identically to normal yaml dicts: their order is purely for readability)
     """
 
-    # We can't control how many ancestors this dumper API uses.
-    # pylint: disable=too-many-ancestors
-    class OrderedDumper(yaml.SafeDumper):
-        pass
-
-    def _dict_representer(dumper, data):
-        return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
-
-    def _range_representer(dumper, data):
-        # type: (yaml.Dumper, Range) -> Node
-        begin, end = data
-
-        # pyyaml doesn't output timestamps in flow style as timestamps(?)
-        if isinstance(begin, datetime.datetime):
-            begin = begin.isoformat()
-        if isinstance(end, datetime.datetime):
-            end = end.isoformat()
-
-        return dumper.represent_mapping(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-            (('begin', begin), ('end', end)),
-            flow_style=True
-        )
-
-    def _reduced_accuracy_decimal_representer(dumper, data):
-        # type: (yaml.Dumper, Decimal) -> Node
-        return dumper.represent_float(
-            float(data)
-        )
-
-    OrderedDumper.add_representer(OrderedDict, _dict_representer)
-    OrderedDumper.add_representer(Range, _range_representer)
-    OrderedDumper.add_representer(Decimal, _reduced_accuracy_decimal_representer)
-    return yaml.dump_all(infos, sys.stdout, OrderedDumper, default_flow_style=False, indent=4)
-
+    return yaml.dump_all(infos, sys.stdout, SafeDatacubeDumper, default_flow_style=False, indent=4)
 
 _OUTPUT_WRITERS = {
     'csv': _write_csv,
     'yaml': _write_yaml,
 }
 
-
 @dataset_cmd.command('info', help="Display dataset information")
 @click.option('--show-sources', help='Also show source datasets', is_flag=True, default=False)
 @click.option('--show-derived', help='Also show derived datasets', is_flag=True, default=False)

datacube/scripts/product.py

@@ -8,10 +8,19 @@
 import sys
 from click import echo
 
+from collections import OrderedDict
+import yaml
+import yaml.resolver
+from yaml import Node
+import csv
+from datacube.api.core import dataset_type_to_row
+import pandas as pd
+
 from datacube.index.index import Index
 from datacube.ui import click as ui
 from datacube.ui.click import cli
 from datacube.utils import read_documents, InvalidDocException
+from datacube.utils.serialise import SafeDatacubeDumper
 
 _LOG = logging.getLogger('datacube-product')
 
@@ -104,44 +113,94 @@ def update_products(index, allow_unsafe, allow_exclusive_lock, dry_run, files):
                                                                                  len(safe_changes)))
     sys.exit(failures)
 
-
-@product.command('list')
-@click.option('--names', is_flag=True, default=False, help='Output product names only, one per line.')
-@ui.pass_datacube()
-def list_products(dc, simple):
+def build_product_list(index):
+    lstdct = []
+    for line in index.products.search():
+        info = dataset_type_to_row(line)
+        lstdct.append(info)
+    return lstdct
+
+def _write_csv(index):
+    writer = csv.DictWriter(sys.stdout, ['id', 'name', 'description',\
+                            'ancillary_quality', 'latgqa_cep90', 'product_type',\
+                            'gqa_abs_iterative_mean_xy', 'gqa_ref_source', 'sat_path',\
+                            'gqa_iterative_stddev_xy', 'time', 'sat_row', 'orbit', 'gqa',\
+                            'instrument', 'gqa_abs_xy', 'crs', 'resolution', 'tile_size',\
+                            'spatial_dimensions'], extrasaction='ignore')
+    writer.writeheader()
+
+    def add_first_name(row):
+        names_ = row['name']
+        row['name'] = names_ if names_ else None
+        return row
+
+    writer.writerows(add_first_name(row) for row in index)
+
+def _write_yaml(index):
     """
-    List products that are defined in the generic index.
+    Dump yaml data with support for OrderedDicts.
+
+    Allows for better human-readability of output: such as dataset ID field first, sources last.
+
+    (Ordered dicts are output identically to normal yaml dicts: their order is purely for readability)
     """
-    products = dc.list_products()
+
+    try:
+        return yaml.dump_all(index, sys.stdout, Dumper=SafeDatacubeDumper, default_flow_style=False, indent=4)
+    except TypeError:
+        return yaml.dump(index.definition, sys.stdout, Dumper=SafeDatacubeDumper, default_flow_style=False, indent=4)
+
+def _write_tab(products):
+    products = pd.DataFrame(products)
 
     if products.empty:
         echo('No products discovered :(')
         return
 
-    if simple:
-        echo('\n'.join(list(products['names'])))
-    echo(products.to_string(columns=('name', 'description', 'product_type', 'instrument',
-                                     'format', 'platform'),
+    echo(products.to_string(columns=('id', 'name', 'description', 'ancillary_quality',\
+                                     'product_type', 'gqa_abs_iterative_mean_xy',\
+                                     'gqa_ref_source', 'sat_path',\
+                                     'gqa_iterative_stddev_xy', 'time', 'sat_row',\
+                                     'orbit', 'gqa', 'instrument', 'gqa_abs_xy', 'crs',\
+                                     'resolution', 'tile_size', 'spatial_dimensions'),
                             justify='left'))
 
+LIST_OUTPUT_WRITERS = {
+    'csv': _write_csv,
+    'yaml': _write_yaml,
+    'tab': _write_tab,
+}
 
-@product.command('show')
-@click.argument('product_name', nargs=1)
-@ui.pass_index()
-def show_product(index, product_name):
+@product.command('list')
+@click.option('-f', help='Output format',
+              type=click.Choice(list(LIST_OUTPUT_WRITERS)), default='yaml', show_default=True)
+@ui.pass_datacube()
+def list_products(dc, f):
     """
-    Show details about a product in the generic index.
+    List products that are defined in the generic index.
     """
+    LIST_OUTPUT_WRITERS[f](build_product_list(dc.index))
+
+def build_product_show(index, product_name):
     product_def = index.products.get_by_name(product_name)
+    return product_def
+
+def _write_json(product_def):
     click.echo_via_pager(json.dumps(product_def.definition, indent=4))
 
 
-@product.command('export')
-@ui.pass_index()
-def export_products(index):
-    """Export all products into YAML."""
-    import yaml
+SHOW_OUTPUT_WRITERS = {
+    'yaml': _write_yaml,
+    'json': _write_json,
+}
 
-    all_products = index.products.get_all()
-    all_product_definitions = (product.definition for product in all_products)
-    click.echo_via_pager(yaml.dump_all(all_product_definitions))
+@product.command('show')
+@click.option('-f', help='Output format',
+              type=click.Choice(list(SHOW_OUTPUT_WRITERS)), default='yaml', show_default=True)
+@click.argument('product_name', nargs=1)
+@ui.pass_datacube()
+def show_product(dc, product_name, f):
+    """
+    Show details about a product in the generic index.
+    """
+    SHOW_OUTPUT_WRITERS[f](build_product_show(dc.index, product_name))

datacube/scripts/user.py

@@ -2,12 +2,22 @@
 
 import logging
 import click
+import csv
+import sys
+import yaml
+import yaml.resolver
+from yaml import Node
+from decimal import Decimal
+
+from collections import OrderedDict
 
 from datacube.utils import gen_password
 from datacube.config import LocalConfig
 from datacube.index.index import Index
 from datacube.ui import click as ui
 from datacube.ui.click import cli
+from datacube.model import Range
+from datacube.utils.serialise import SafeDatacubeDumper
 
 _LOG = logging.getLogger('datacube-user')
 USER_ROLES = ('user', 'ingest', 'manage', 'admin')
@@ -17,16 +27,57 @@
 def user_cmd():
     pass
 
+def build_user_list(index):
+    lstdct = []
+    for role, user, description in index.users.list_users():
+        info = OrderedDict((
+            ('role', role),
+            ('user', user),
+            ('description', description)
+        ))
+        lstdct.append(info)
+    return lstdct
+
+def _write_csv(index):
+    writer = csv.DictWriter(sys.stdout, ['role', 'user', 'description'], extrasaction='ignore')
+    writer.writeheader()
+
+    def add_first_role(row):
+        roles_ = row['role']
+        row['role'] = roles_ if roles_ else None
+        return row
+
+    writer.writerows(add_first_role(row) for row in index)
+
+
+def _write_yaml(index):
+    """
+    Dump yaml data with support for OrderedDicts.
+
+    Allows for better human-readability of output: such as dataset ID field first, sources last.
+
+    (Ordered dicts are output identically to normal yaml dicts: their order is purely for readability)
+    """
+
+    return yaml.dump_all(index, sys.stdout, SafeDatacubeDumper, default_flow_style=False, indent=4)
+
+_OUTPUT_WRITERS = {
+    'csv': _write_csv,
+    'yaml': _write_yaml,
+}
+
 
 @user_cmd.command('list')
+@click.option('-f', help='Output format',
+              type=click.Choice(list(_OUTPUT_WRITERS)), default='yaml', show_default=True)
 @ui.pass_index()
-def list_users(index):
+
+
+def list_users(index, f):
     """
     List users
     """
-    for role, user, description in index.users.list_users():
-        click.echo('{0:6}\t{1:15}\t{2}'.format(role, user, description if description else ''))
-
+    _OUTPUT_WRITERS[f](build_user_list(index))
 
 @user_cmd.command('grant')
 @click.argument('role',

datacube/utils/serialise.py

@@ -0,0 +1,43 @@
+# coding=utf-8
+"""
+Serialise function used in YAML output
+"""
+from __future__ import absolute_import, division, print_function
+
+import os
+from collections import OrderedDict
+from datetime import datetime, date
+from decimal import Decimal
+
+import yaml
+from datacube.model import Range
+
+class SafeDatacubeDumper(yaml.SafeDumper): # pylint: disable=too-many-ancestors
+    pass
+def _dict_representer(dumper, data):
+    return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
+
+def _range_representer(dumper, data):
+    # type: (yaml.Dumper, Range) -> Node
+    begin, end = data
+
+    # pyyaml doesn't output timestamps in flow style as timestamps(?)
+    if isinstance(begin, datetime):
+        begin = begin.isoformat()
+    if isinstance(end, datetime):
+        end = end.isoformat()
+
+    return dumper.represent_mapping(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        (('begin', begin), ('end', end)),
+        flow_style=True
+    )
+
+def _reduced_accuracy_decimal_representer(dumper, data):
+    # type: (yaml.Dumper, Decimal) -> Node
+    return dumper.represent_float(
+        float(data)
+    )
+SafeDatacubeDumper.add_representer(OrderedDict, _dict_representer)
+SafeDatacubeDumper.add_representer(Range, _range_representer)
+SafeDatacubeDumper.add_representer(Decimal, _reduced_accuracy_decimal_representer)

docs/about/whats_new.rst

@@ -49,6 +49,8 @@ Next release
 
  - Added Dockerfile to enable with automated builds for a reference Docker image.
 
+ - Command line tools can now output CSV or YAML. (Issue #206, PR 390)
+
 .. _#298: https://github.com/opendatacube/datacube-core/pull/298
 .. _config docs: https://datacube-core.readthedocs.io/en/latest/ops/config.html#runtime-config-doc