Py3 server with + event/cmd cron jobs processing with django 3.0

NOTES.add_event_type

@@ -34,14 +34,14 @@ export KADI=$PWD
 
 ./manage.py syncdb
 
-./update_events --start=1999:200 --stop=2001:001 --model=${ModelClassName}
+python -m kadi.update_events --start=1999:200 --stop=2001:001 --model=${ModelClassName}
 # Update early events first and look for warnings.  Also confirm that the first event
 # matches what is in the current database unless a change is intended.
 # Some events may need a later start data to be fully sampled.  Manvr, for example,
 # should use a start of "1999:251"  (1999:230 + 21 days lookback).
 # Probably not needed for events that rely on only one event MSID.
 
-./update_events --start=2001:001 --model=${ModelClassName}
+python -m kadi.update_events --start=2001:001 --model=${ModelClassName}
 # Update the rest of the time range for the updated model/event
 
 
@@ -104,7 +104,7 @@ Use the current flight kadi code and the test database and confirm that kadi tes
   In [9]: kadi.test()
   ============================= test session starts ==============================
   platform linux2 -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4
-  collected 15 items 
+  collected 15 items
 
   kadi/tests/test_events.py ..........
   kadi/tests/test_occweb.py .....

NOTES.build

@@ -4,37 +4,28 @@
 
 cd ~/git/kadi
 export KADI=$PWD
-rm events.db3 cmds.h5 cmds.pkl
-./manage.py syncdb
-
-export ENG_ARCHIVE=/proj/sot/ska/data/eng_archive/1999
-./update_events --start=1999:200 --stop=2000:001 --model=SafeSun
-
-export ENG_ARCHIVE=
-./update_events --start=2000:001
-./update_cmds --start=2000:001
-
-NOTE: could potentially get all the 1999 events using the 1999 eng
-      archive.
+rm -f events3.db3 cmds.h5 cmds.pkl
+rm -rf kadi/events/migrations
+./manage.py makemigrations events
+./manage.py migrate
+
+# For commands one MUST do this in a dedicated test env because the pickling
+# of UpdatedDict does not work.  That object gets a module of __main__ but for
+# production it must be kadi.update_cmds.  See e.g.
+# https://www.stefaanlippens.net/python-pickling-and-dealing-with-attributeerror-
+# module-object-has-no-attribute-thing.html
+pip install .  # to a TEST env!!  (Maybe with -e for editable install?)
+
+# First line is just to see that every model works.  One can just drop the
+# --stop=2000:001 if you are sure it will work.
+kadi_update_events --start=1999:240 --stop=2000:001
+kadi_update_events --start=2000:001
+kadi_update_cmds --start=2000:001
 
 #################################################################
-# Remove certain tables in a local version of the flight events database
+# Re-build single table
 #################################################################
 
 % export KADI=$PWD
 % cp /proj/sot/ska/data/kadi/events.db3 ./
-% sqlite3 events.db3
-> drop table events_cap;
-
-% ./manage.py syncdb
-% ./update_events --start=2000:001 --model=CAP --delete-from-start
-
-
-#################################################################
-# Reprocess a table from a starting date in a local version of
-# the flight events database
-#################################################################
-
-% cp /proj/sot/ska/data/kadi/events.db3 ./
-% ./update_events --start=2012:001 --model=Scs107 --delete-from-start
-
+% python -m kadi.update_events --start=1999:001 --model=CAP --delete-from-start

NOTES.reprocess

@@ -11,6 +11,6 @@ If the kadi events.db3 database gets corrupted (e.g. problems related to the
   cp /proj/sot/ska/data/kadi/events.db3 ./
 
   # Update from 2017:017 (for example) to present
-  ./update_events --start=2017:017 --delete-from-start
+  python -m kadi.update_events --start=2017:017 --delete-from-start
 
   cp events.db3 /proj/sot/ska/data/kadi/

NOTES.update_ltt_bads

@@ -8,7 +8,7 @@ Test
 ----
 
 export KADI=$PWD
-./update_events --start '1999:100' --delete-from-start --loop-days=10000 \
+python -m kadi.update_events --start '1999:100' --delete-from-start --loop-days=10000 \
                 --model=LttBad >& run.log &
 
 # Confirm new events added as expected
@@ -23,7 +23,7 @@ make install  # or just copy ltt_bads.dat to $ska/data/kadi/.
 
 unset KADI
 cd $ska/share/kadi
-./update_events --start '1999:100' --delete-from-start --loop-days=10000 \
+python -m kadi.update_events --start '1999:100' --delete-from-start --loop-days=10000 \
                 --data-root=$ska/data/kadi --model=LttBad >& run.log &
 
 

docs/conf.py

@@ -62,7 +62,7 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-from kadi.version import version as release
+from kadi import __version__ as release
 # The short X.Y version.
 version = '.'.join(release.split('.')[:2])
 

kadi/__init__.py

@@ -1,5 +1,8 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-from .version import __version__, __git_version__  # noqa
+
+import ska_helpers
+
+__version__ = ska_helpers.get_version(__package__)
 
 
 def test(*args, **kwargs):

kadi/cmds/cmds.py

@@ -5,8 +5,7 @@
 
 from astropy.table import Table
 from Chandra.Time import DateTime
-import six
-from six.moves import cPickle as pickle
+import pickle
 
 from ..paths import IDX_CMDS_PATH, PARS_DICT_PATH
 
@@ -51,8 +50,7 @@ def load_idx_cmds():
 
 def load_pars_dict():
     with open(PARS_DICT_PATH(), 'rb') as fh:
-        kwargs = {} if six.PY2 else {'encoding': 'ascii'}
-        pars_dict = pickle.load(fh, **kwargs)
+        pars_dict = pickle.load(fh, encoding='ascii')
     return pars_dict
 
 
@@ -141,7 +139,7 @@ def _find(start=None, stop=None, **kwargs):
         ok &= idx_cmds['date'] < DateTime(stop).date
     for key, val in kwargs.items():
         key = key.lower()
-        if isinstance(val, six.string_types):
+        if isinstance(val, str):
             val = val.upper()
         if key in idx_cmds.dtype.names:
             ok &= idx_cmds[key] == val
@@ -173,17 +171,17 @@ def __init__(self, cmd):
         if self['tlmsid'] == 'None':
             colnames.remove('tlmsid')
 
-        self._ordered_keys = (colnames[1:] +
-                              [par[0] for par in rev_pars_dict[cmd['idx']]])
+        self._ordered_keys = (colnames[1:]
+                              + [par[0] for par in rev_pars_dict[cmd['idx']]])
 
     def __repr__(self):
         out = ('<{} '.format(self.__class__.__name__) + str(self) + '>')
         return out
 
     def __str__(self):
-        out = ('{} {:11s} '.format(self['date'], self['type']) +
-               ' '.join('{}={}'.format(key, self[key]) for key in self._ordered_keys
-                        if key not in ('type', 'date')))
+        out = ('{} {:11s} '.format(self['date'], self['type'])
+               + ' '.join('{}={}'.format(key, self[key]) for key in self._ordered_keys
+                          if key not in ('type', 'date')))
         return out
 
 
@@ -203,7 +201,7 @@ def __len__(self):
 
     def __getitem__(self, item):
         cmds = self.cmds
-        if isinstance(item, six.string_types):
+        if isinstance(item, str):
             if item in cmds.colnames:
                 return cmds[item]
 

kadi/commands/commands.py

@@ -6,8 +6,7 @@
 
 from astropy.table import Table, Row, Column, vstack
 from Chandra.Time import DateTime
-import six
-from six.moves import cPickle as pickle
+import pickle
 
 from ..paths import IDX_CMDS_PATH, PARS_DICT_PATH
 
@@ -52,8 +51,7 @@ def load_idx_cmds():
 
 def load_pars_dict():
     with open(PARS_DICT_PATH(), 'rb') as fh:
-        kwargs = {} if six.PY2 else {'encoding': 'ascii'}
-        pars_dict = pickle.load(fh, **kwargs)
+        pars_dict = pickle.load(fh, encoding='ascii')
     return pars_dict
 
 
@@ -195,7 +193,7 @@ def _find(start=None, stop=None, **kwargs):
         ok &= idx_cmds['date'] < DateTime(stop).date
     for key, val in kwargs.items():
         key = key.lower()
-        if isinstance(val, six.string_types):
+        if isinstance(val, str):
             val = val.upper()
         if key in idx_cmds.dtype.names:
             ok &= idx_cmds[key] == val
@@ -246,9 +244,9 @@ def __str__(self):
         if 'idx' in keys:
             keys.remove('idx')
 
-        out = ('{} {} '.format(self['date'], self['type']) +
-               ' '.join('{}={}'.format(key, self[key]) for key in keys
-                        if key not in ('type', 'date')))
+        out = ('{} {} '.format(self['date'], self['type'])
+               + ' '.join('{}={}'.format(key, self[key]) for key in keys
+                          if key not in ('type', 'date')))
         return out
 
     def __sstr__(self):
@@ -262,7 +260,7 @@ class CommandTable(Table):
     """
 
     def __getitem__(self, item):
-        if isinstance(item, six.string_types):
+        if isinstance(item, str):
             if item in self.colnames:
                 return self.columns[item]
             else:
@@ -280,11 +278,11 @@ def __getitem__(self, item):
                                              keys=self.groups._keys)
             return out
 
-        elif (isinstance(item, slice) or
-              isinstance(item, np.ndarray) or
-              isinstance(item, list) or
-              isinstance(item, tuple) and all(isinstance(x, np.ndarray)
-                                              for x in item)):
+        elif (isinstance(item, slice)
+              or isinstance(item, np.ndarray)
+              or isinstance(item, list)
+              or isinstance(item, tuple) and all(isinstance(x, np.ndarray)
+                                                 for x in item)):
             # here for the many ways to give a slice; a tuple of ndarray
             # is produced by np.where, as in t[np.where(t['a'] > 2)]
             # For all, a new table is constructed with slice of all columns
@@ -294,7 +292,7 @@ def __getitem__(self, item):
             raise ValueError('Illegal type {0} for table item access'
                              .format(type(item)))
 
-    def __unicode__(self):
+    def __str__(self):
         # Cut out params column for printing
         colnames = self.colnames
         if 'idx' in colnames:
@@ -320,13 +318,8 @@ def __unicode__(self):
         lines = tmp.pformat(max_width=-1)
         return '\n'.join(lines)
 
-    if not six.PY2:
-        __str__ = __unicode__
-
     def __bytes__(self):
-        return six.text_type(self).encode('utf-8')
-    if six.PY2:
-        __str__ = __bytes__
+        return str(self).encode('utf-8')
 
     def fetch_params(self):
         """

kadi/commands/states.py

@@ -9,8 +9,6 @@
 import inspect
 
 import numpy as np
-import six
-from six.moves import range
 
 from astropy.table import Table, Column
 
@@ -40,10 +38,10 @@
 
 # State keys for PCAD-related transitions.  If *any* of these are requested then
 # *all* of them need to be processed to get the correct answer.
-PCAD_STATE_KEYS = (QUAT_COMPS +
-                   ['targ_' + qc for qc in QUAT_COMPS] +
-                   ['ra', 'dec', 'roll'] +
-                   ['auto_npnt', 'pcad_mode', 'pitch', 'off_nom_roll'])
+PCAD_STATE_KEYS = (QUAT_COMPS
+                   + ['targ_' + qc for qc in QUAT_COMPS]
+                   + ['ra', 'dec', 'roll']
+                   + ['auto_npnt', 'pcad_mode', 'pitch', 'off_nom_roll'])
 
 # Default state keys (mostly matches classic command states list)
 DEFAULT_STATE_KEYS = ('ccd_count', 'clocking', 'dec', 'dither', 'fep_count',
@@ -114,12 +112,10 @@ def __new__(mcls, name, bases, members):
         return cls
 
 
-@six.add_metaclass(TransitionMeta)
-class BaseTransition(object):
+class BaseTransition(object, metaclass=TransitionMeta):
     """
     Base transition class from which all actual transition classes are derived.
     """
-
     @classmethod
     def get_state_changing_commands(cls, cmds):
         """
@@ -192,8 +188,8 @@ def _auto_update_docstring(cls):
 
         others = []
         for attr, val in cls.__dict__.items():
-            if (attr.startswith('_') or inspect.ismethod(val) or
-                    attr in ('state_keys', 'command_attributes', 'command_params')):
+            if (attr.startswith('_') or inspect.ismethod(val)
+                    or attr in ('state_keys', 'command_attributes', 'command_params')):
                 continue
             others.append('{}={}'.format(attr, val))
         if others:
@@ -992,7 +988,7 @@ def get_transition_classes(state_keys=None):
     Get all BaseTransition subclasses in this module corresponding to
     state keys ``state_keys``.
     """
-    if isinstance(state_keys, six.string_types):
+    if isinstance(state_keys, str):
         state_keys = [state_keys]
     elif state_keys is None:
         state_keys = DEFAULT_STATE_KEYS

kadi/commands/tests/test_states.py

@@ -107,10 +107,10 @@ def test_quick():
     Test for a few days in 2017.  Sanity check for refactoring etc.
     """
     state_keys = (['obsid', 'clocking', 'power_cmd', 'fep_count', 'vid_board',
-                   'si_mode', 'ccd_count'] +
-                  ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] +
-                  ['letg', 'hetg'] +
-                  ['simpos', 'simfa_pos'])
+                   'si_mode', 'ccd_count']
+                  + ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll']
+                  + ['letg', 'hetg']
+                  + ['simpos', 'simfa_pos'])
     continuity = {'letg': 'RETR', 'hetg': 'RETR'}  # Not necessarily set within 7 days
     rc, rk = compare_states('2018:235', '2018:245', state_keys, continuity=continuity)
 
@@ -154,6 +154,7 @@ def test_states_2017():
     maneuver commanding (so the NMAN period before maneuver starts is OK) while kadi
     will insert pitch breaks only in NPNT.  (Tested later).
     """
+
     state_keys = (['obsid', 'clocking', 'power_cmd', 'fep_count']
                   + ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll']
                   + ['letg', 'hetg']
@@ -442,8 +443,8 @@ def test_reduce_states_cmd_states():
     cs = cmd_states.fetch_states('2018:235', '2018:245', allow_identical=True)
     cs = Table(cs)
 
-    state_keys = (set(cmd_states.STATE0) -
-                  set(['datestart', 'datestop', 'trans_keys', 'tstart', 'tstop']))
+    state_keys = (set(cmd_states.STATE0)
+                  - set(['datestart', 'datestop', 'trans_keys', 'tstart', 'tstop']))
 
     # Default setting is reduce states with merge_identical=False, which is the same
     # as cmd_states.

kadi/events/__init__.py

@@ -44,16 +44,12 @@
 import os
 import django
 
+# If we are running standalone then this ENV var is not set.  Need to explicitly
+# do the setup and then import query module attributes for standalone.
+# For WSGI server the env var is set in wsgi.py.
+# For the dev server it is set in manage.py.
+
 if 'DJANGO_SETTINGS_MODULE' not in os.environ:
     os.environ['DJANGO_SETTINGS_MODULE'] = 'kadi.settings'
-
-# In django >= 1.8 an explicit call to setup is required for standalone.
-# This attribute does not exist in 1.6.
-# https://docs.djangoproject.com/en/1.10/topics/settings/#calling-django-setup-is-required-for-standalone-django-usage
-try:
     django.setup()
-except AttributeError:
-    pass
-
-# from .models import *
-from .query import *  # noqa
+    from .query import *  # noqa