diff --git a/Ska/engarchive/fetch.py b/Ska/engarchive/fetch.py index 4dae7433..9c715182 100644 --- a/Ska/engarchive/fetch.py +++ b/Ska/engarchive/fetch.py @@ -3,8 +3,6 @@ """ Fetch values from the Ska engineering telemetry archive. """ -from __future__ import print_function, division, absolute_import - import sys import os import time @@ -382,8 +380,8 @@ def msid_glob(msid): :param msid: input MSID glob :returns: tuple (msids, MSIDs) """ - msids = collections.OrderedDict() - MSIDS = collections.OrderedDict() + msids = {} + MSIDS = {} # First check if `msid` matches a computed class. This does not allow # for globs, and here the output MSIDs is the single computed class. @@ -416,10 +414,25 @@ def _msid_glob(msid, source): """ source_msids = data_source.get_msids(source) + # MSID is the upper-case version of the MSID name that is actually used in + # backend queries. ``msid`` is the user-supplied version. MSID = msid.upper() - # First try MSID or DP_. If success then return the upper + + # CALC_ is a synonym for DP_ which works in both CXC and MAUDE archives, so + # swap to DP_ if CALC_ is found. These are calculated pseudo-MSIDs. + if MSID.startswith('CALC_'): + MSID = 'DP_' + MSID[5:] + + # matches_msid is a list of MSIDs that match the input MSID. Providing the + # initial DP_ is optional so we try both if the MSID doesn't already start + # with DP_ (i.e. PITCH or DP_PITCH). + matches_msid = (MSID,) + if not MSID.startswith('DP_'): + matches_msid += ('DP_' + MSID,) + + # If one of matches_msid is in the source then return the upper # case version and whatever the user supplied (could be any case). - for match in (MSID, 'DP_' + MSID): + for match in matches_msid: if match in source_msids: return [msid], [match] @@ -427,7 +440,7 @@ def _msid_glob(msid, source): # list of matches, all lower case and all upper case. Since the # input was a glob the returned msids are just lower case versions # of the matched upper case MSIDs. - for match in (MSID, 'DP_' + MSID): + for match in matches_msid: matches = fnmatch.filter(source_msids, match) if matches: if len(matches) > MAX_GLOB_MATCHES: @@ -1401,7 +1414,11 @@ def plot(self, *args, **kwargs): plot_cxctime(self.times, vals, *args, state_codes=self.state_codes, **kwargs) plt.margins(0.02, 0.05) - plt.title(self.MSID) + # Upper-cased version of msid name from user + title = self.msid.upper() + if self.stat: + title = f'{title} ({self.stat})' + plt.title(title) if self.unit: plt.ylabel(self.unit) diff --git a/Ska/engarchive/plot.py b/Ska/engarchive/plot.py index 0f7ac3d4..13e95fdf 100644 --- a/Ska/engarchive/plot.py +++ b/Ska/engarchive/plot.py @@ -206,7 +206,10 @@ def draw_plot(self): dy = min(ymin + ymax, 1e-12) * 0.05 self.ax.set_ylim(ymin - dy, ymax + dy) - self.ax.set_title('{} {}'.format(msid.MSID, msid.stat or '')) + title = msid.msid.upper() + if msid.stat: + title = f'{title} ({msid.stat})' + self.ax.set_title(title) if msid.unit: self.ax.set_ylabel(msid.unit) diff --git a/Ska/engarchive/tests/test_fetch.py b/Ska/engarchive/tests/test_fetch.py index 5710720d..84764a15 100644 --- a/Ska/engarchive/tests/test_fetch.py +++ b/Ska/engarchive/tests/test_fetch.py @@ -123,6 +123,22 @@ def test_filter_bad_times_default_copy(): assert np.all(dates == DATES_EXPECT2) +@pytest.mark.parametrize('msid', ['DP_piTch_fss', 'Calc_pitCH_fss']) +@pytest.mark.parametrize('sources', (('cxc',), ('maude',), ('cxc', 'maude'))) +def test_fetch_derived_param_aliases(msid, sources): + cxc_tstop = fetch.get_time_range('dp_pitch_fss', 'secs')[1] + with fetch.data_source(*sources): + # Get data from 2 days to present to ensure MAUDE is queried + dt = 200 # seconds + d1 = fetch.Msid('piTch_fss', cxc_tstop - dt, cxc_tstop + dt) + d2 = fetch.Msid(msid, cxc_tstop - dt, cxc_tstop + dt) + assert d2.msid == msid # version as the user provide + assert d2.MSID == d1.MSID # normalized version for accessing databases + assert np.all(d1.times == d2.times) + assert np.all(d1.vals == d2.vals) + assert len(d1) > 0 + + def test_interpolate(): dat = fetch.MSIDset(['aoattqt1', 'aogyrct1', 'aopcadmd'], '2008:002:21:48:00', '2008:002:21:50:00') diff --git a/docs/fetch_tutorial.rst b/docs/fetch_tutorial.rst index b83f3400..33e4b394 100644 --- a/docs/fetch_tutorial.rst +++ b/docs/fetch_tutorial.rst @@ -63,6 +63,26 @@ a stop time then it defaults to the latest available data in the archive. tephin = fetch.Msid('tephin', '2010:001') # Same as previous tephin = fetch.Msid('tephin') # Launch through NOW +**Derived parameters (calcs)** + +The engineering archive has pseudo-MSIDs that are derived via calculation from +telemetry MSIDs. These are also known as "calcs" in the context of MAUDE. In +MAUDE, a calc is normally indicated with a prefix of ``CALC_``, but for +compatibility with cheta a prefix of ``DP_`` is also allowed. + +Derived parameter names begin with the characters ``DP_`` (not case sensitive as +usual). Otherwise there is no difference from standard MSIDs. When querying +the archive using ``fetch``, there are three equivalent ways to specify an +MSID name: + +- ``DP_>`` e.g. ``DP_PITCH_FSS`` +- ``CALC_`` e.g. ``CALC_PITCH_FSS`` +- ```` e.g. ``PITCH_FSS``: this is a convenience and internally fetch will + search for derived parameters matching ``DP_``. + +See the :ref:`derived-parameters-or-calcs` section for more details and full +listings of available MSIDs. + **Other details** If you are wondering what time range of data is available for a particular MSID @@ -1041,8 +1061,7 @@ The key differences between the CXC and MAUDE telemetry data sources are: - CXC includes `pseudo-MSIDs <../pseudo_msids.html>`_ such as ephemeris data, ACIS and HRC housekeeping, and derived parameters like the pitch and off-nominal roll angle. - CXC has a latency of 2-3 days vs. hours for MAUDE back-orbit telemetry. -- During a realtime support MAUDE provides near-realime telemetry. -- As of MAUDE 0.7.2 there is no support for 5-minute and daily stats (coming in 0.7.3). +- During a realtime support MAUDE provides real-time telemetry. - CXC has about 6800 MSIDs while MAUDE has around 11350. At least some of the MSIDs that are only in MAUDE are somewhat obscure ones like ``ACIMG1D1`` (PEA1 PIXEL D1 DATA IMAGE 1) which the CXC decoms into higher-level products. @@ -1050,8 +1069,9 @@ The key differences between the CXC and MAUDE telemetry data sources are: system memory (gigabytes) and **always returns all available data points**. - MAUDE is optimized for smaller, more frequent queries and uses a secure web server to provide data. It has limits on both the number of returned data values (around 100k) - and the total number of bytes in the data (around 1.6 Mb). **MAUDE will sub-sample - the data as necessary to fit in the data limits (see below for example)**. + and the total number of bytes in the data (around 1.6 Mb). +- By default MAUDE will sub-sample the data as necessary to fit in the data limits + (see below for example of disabling this feature). Basic usage ----------- diff --git a/docs/pseudo_msids.rst b/docs/pseudo_msids.rst index 0496b76c..16c5a19e 100644 --- a/docs/pseudo_msids.rst +++ b/docs/pseudo_msids.rst @@ -565,14 +565,38 @@ SCCT4 Sci single detector counter CT4 SCCT5 Sci single detector counter CT5 ==================== ====== ========================================= +.. _derived-parameters-or-calcs: -Derived Parameters ------------------- +Derived Parameters or Calcs +--------------------------- The engineering archive has pseudo-MSIDs that are derived via computation from -telemetry MSIDs. All derived parameter names begin with the characters ``DP_`` -(not case sensitive as usual). Otherwise there is no difference from standard -MSIDs. +telemetry MSIDs. These are also known as "calcs" in the context of MAUDE (which +inherited this from GRETA). In MAUDE, a calc is normally indicated with a prefix +of ``CALC_``, but for compatibility with cheta a prefix of ``DP_`` is also +allowed. + +Derived parameter names begin with the characters ``DP_`` (not case sensitive as +usual). Otherwise there is no difference from standard MSIDs. When querying +the archive using ``fetch``, there are three equivalent ways to specify an +MSID name: + +- ``DP_>`` e.g. ``DP_PITCH_FSS`` +- ``CALC_`` e.g. ``CALC_PITCH_FSS`` +- ```` e.g. ``PITCH_FSS``: this is a convenience and internally ``fetch`` + will search for derived parameters matching ``DP_``. + +Available MSIDs +^^^^^^^^^^^^^^^ + +To see the available derived parameters or calcs in the CXC archive or MAUDE +archive, issue the following commands respectively:: + + >>> from cheta import fetch + >>> sorted([msid for msid in fetch.data_source.get_msids('cxc') + ... if msid.startswith('DP_')]) + >>> sorted([msid for msid in fetch.data_source.get_msids('maude') + ... if msid.startswith('CALC_')]) Definition ^^^^^^^^^^^