Merge pull request #227 from sot/allow-calc-prefix

Allow CALC_ prefix as an alias for DP_

Ska/engarchive/fetch.py

@@ -3,8 +3,6 @@
 """
 Fetch values from the Ska engineering telemetry archive.
 """
-from __future__ import print_function, division, absolute_import
-
 import sys
 import os
 import time
@@ -382,8 +380,8 @@ def msid_glob(msid):
     :param msid: input MSID glob
     :returns: tuple (msids, MSIDs)
     """
-    msids = collections.OrderedDict()
-    MSIDS = collections.OrderedDict()
+    msids = {}
+    MSIDS = {}
 
     # First check if `msid` matches a computed class.  This does not allow
     # for globs, and here the output MSIDs is the single computed class.
@@ -416,18 +414,33 @@ def _msid_glob(msid, source):
     """
     source_msids = data_source.get_msids(source)
 
+    # MSID is the upper-case version of the MSID name that is actually used in
+    # backend queries. ``msid`` is the user-supplied version.
     MSID = msid.upper()
-    # First try MSID or DP_<MSID>.  If success then return the upper
+
+    # CALC_ is a synonym for DP_ which works in both CXC and MAUDE archives, so
+    # swap to DP_ if CALC_ is found. These are calculated pseudo-MSIDs.
+    if MSID.startswith('CALC_'):
+        MSID = 'DP_' + MSID[5:]
+
+    # matches_msid is a list of MSIDs that match the input MSID. Providing the
+    # initial DP_ is optional so we try both if the MSID doesn't already start
+    # with DP_ (i.e. PITCH or DP_PITCH).
+    matches_msid = (MSID,)
+    if not MSID.startswith('DP_'):
+        matches_msid += ('DP_' + MSID,)
+
+    # If one of matches_msid is in the source then return the upper
     # case version and whatever the user supplied (could be any case).
-    for match in (MSID, 'DP_' + MSID):
+    for match in matches_msid:
         if match in source_msids:
             return [msid], [match]
 
     # Next try as a file glob.  If there is a match then return a
     # list of matches, all lower case and all upper case.  Since the
     # input was a glob the returned msids are just lower case versions
     # of the matched upper case MSIDs.
-    for match in (MSID, 'DP_' + MSID):
+    for match in matches_msid:
         matches = fnmatch.filter(source_msids, match)
         if matches:
             if len(matches) > MAX_GLOB_MATCHES:
@@ -1401,7 +1414,11 @@ def plot(self, *args, **kwargs):
         plot_cxctime(self.times, vals, *args, state_codes=self.state_codes,
                      **kwargs)
         plt.margins(0.02, 0.05)
-        plt.title(self.MSID)
+        # Upper-cased version of msid name from user
+        title = self.msid.upper()
+        if self.stat:
+            title = f'{title} ({self.stat})'
+        plt.title(title)
         if self.unit:
             plt.ylabel(self.unit)
 

Ska/engarchive/plot.py

@@ -206,7 +206,10 @@ def draw_plot(self):
                 dy = min(ymin + ymax, 1e-12) * 0.05
             self.ax.set_ylim(ymin - dy, ymax + dy)
 
-        self.ax.set_title('{} {}'.format(msid.MSID, msid.stat or ''))
+        title = msid.msid.upper()
+        if msid.stat:
+            title = f'{title} ({msid.stat})'
+        self.ax.set_title(title)
         if msid.unit:
             self.ax.set_ylabel(msid.unit)
 

Ska/engarchive/tests/test_fetch.py

@@ -151,6 +151,22 @@ def test_filter_bad_times_default_copy():
     assert np.all(dates == DATES_EXPECT2)
 
 
+@pytest.mark.parametrize('msid', ['DP_piTch_fss', 'Calc_pitCH_fss'])
+@pytest.mark.parametrize('sources', (('cxc',), ('maude',), ('cxc', 'maude')))
+def test_fetch_derived_param_aliases(msid, sources):
+    cxc_tstop = fetch.get_time_range('dp_pitch_fss', 'secs')[1]
+    with fetch.data_source(*sources):
+        # Get data from 2 days to present to ensure MAUDE is queried
+        dt = 200  # seconds
+        d1 = fetch.Msid('piTch_fss', cxc_tstop - dt, cxc_tstop + dt)
+        d2 = fetch.Msid(msid, cxc_tstop - dt, cxc_tstop + dt)
+    assert d2.msid == msid  # version as the user provide
+    assert d2.MSID == d1.MSID  # normalized version for accessing databases
+    assert np.all(d1.times == d2.times)
+    assert np.all(d1.vals == d2.vals)
+    assert len(d1) > 0
+
+
 def test_interpolate():
     dat = fetch.MSIDset(['aoattqt1', 'aogyrct1', 'aopcadmd'],
                         '2008:002:21:48:00', '2008:002:21:50:00')

docs/fetch_tutorial.rst

@@ -63,6 +63,26 @@ a stop time then it defaults to the latest available data in the archive.
   tephin = fetch.Msid('tephin', '2010:001') # Same as previous
   tephin = fetch.Msid('tephin') # Launch through NOW
 
+**Derived parameters (calcs)**
+
+The engineering archive has pseudo-MSIDs that are derived via calculation from
+telemetry MSIDs. These are also known as "calcs" in the context of MAUDE. In
+MAUDE, a calc is normally indicated with a prefix of ``CALC_``, but for
+compatibility with cheta a prefix of ``DP_`` is also allowed.
+
+Derived parameter names begin with the characters ``DP_`` (not case sensitive as
+usual).  Otherwise there is no difference from standard MSIDs. When querying
+the archive using ``fetch``, there are three equivalent ways to specify an
+MSID name:
+
+- ``DP_<name>>`` e.g. ``DP_PITCH_FSS``
+- ``CALC_<name>`` e.g. ``CALC_PITCH_FSS``
+- ``<name>`` e.g. ``PITCH_FSS``: this is a convenience and internally fetch will
+  search for derived parameters matching ``DP_<name>``.
+
+See the :ref:`derived-parameters-or-calcs` section for more details and full
+listings of available MSIDs.
+
 **Other details**
 
 If you are wondering what time range of data is available for a particular MSID
@@ -1041,17 +1061,17 @@ The key differences between the CXC and MAUDE telemetry data sources are:
 - CXC includes `pseudo-MSIDs <../pseudo_msids.html>`_ such as ephemeris data, ACIS and HRC
   housekeeping, and derived parameters like the pitch and off-nominal roll angle.
 - CXC has a latency of 2-3 days vs. hours for MAUDE back-orbit telemetry.
-- During a realtime support MAUDE provides near-realime telemetry.
-- As of MAUDE 0.7.2 there is no support for 5-minute and daily stats (coming in 0.7.3).
+- During a realtime support MAUDE provides real-time telemetry.
 - CXC has about 6800 MSIDs while MAUDE has around 11350.  At least some of the MSIDs that
   are only in MAUDE are somewhat obscure ones like ``ACIMG1D1`` (PEA1 PIXEL D1 DATA IMAGE
   1) which the CXC decoms into higher-level products.
 - CXC is optimized for large bulk queries using direct disk access.  It is limited only by
   system memory (gigabytes) and **always returns all available data points**.
 - MAUDE is optimized for smaller, more frequent queries and uses a secure web server to
   provide data.  It has limits on both the number of returned data values (around 100k)
-  and the total number of bytes in the data (around 1.6 Mb).  **MAUDE will sub-sample
-  the data as necessary to fit in the data limits (see below for example)**.
+  and the total number of bytes in the data (around 1.6 Mb).
+- By default MAUDE will sub-sample the data as necessary to fit in the data limits
+  (see below for example of disabling this feature).
 
 Basic usage
 -----------

docs/pseudo_msids.rst

@@ -565,14 +565,38 @@ SCCT4                        Sci single detector counter CT4
 SCCT5                        Sci single detector counter CT5
 ==================== ====== =========================================
 
+.. _derived-parameters-or-calcs:
 
-Derived Parameters
-------------------
+Derived Parameters or Calcs
+---------------------------
 
 The engineering archive has pseudo-MSIDs that are derived via computation from
-telemetry MSIDs.  All derived parameter names begin with the characters ``DP_``
-(not case sensitive as usual).  Otherwise there is no difference from standard
-MSIDs.
+telemetry MSIDs. These are also known as "calcs" in the context of MAUDE (which
+inherited this from GRETA). In MAUDE, a calc is normally indicated with a prefix
+of ``CALC_``, but for compatibility with cheta a prefix of ``DP_`` is also
+allowed.
+
+Derived parameter names begin with the characters ``DP_`` (not case sensitive as
+usual).  Otherwise there is no difference from standard MSIDs. When querying
+the archive using ``fetch``, there are three equivalent ways to specify an
+MSID name:
+
+- ``DP_<name>>`` e.g. ``DP_PITCH_FSS``
+- ``CALC_<name>`` e.g. ``CALC_PITCH_FSS``
+- ``<name>`` e.g. ``PITCH_FSS``: this is a convenience and internally ``fetch``
+  will search for derived parameters matching ``DP_<name>``.
+
+Available MSIDs
+^^^^^^^^^^^^^^^
+
+To see the available derived parameters or calcs in the CXC archive or MAUDE
+archive, issue the following commands respectively::
+
+   >>> from cheta import fetch
+   >>> sorted([msid for msid in fetch.data_source.get_msids('cxc')
+   ...        if msid.startswith('DP_')])
+   >>> sorted([msid for msid in fetch.data_source.get_msids('maude')
+   ...        if msid.startswith('CALC_')])
 
 Definition
 ^^^^^^^^^^^