From ca64fa5e0e04329a517f467139cf37d2bd0baa04 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Mon, 16 Dec 2019 16:20:49 -0500 Subject: [PATCH 01/35] Changes to use scm_setuptools --- kadi/version.py | 111 ++++++++---------------------------------------- setup.py | 3 +- 2 files changed, 19 insertions(+), 95 deletions(-) diff --git a/kadi/version.py b/kadi/version.py index c9cec6b4..17af2df5 100644 --- a/kadi/version.py +++ b/kadi/version.py @@ -17,8 +17,6 @@ ``version_object`` has a number of useful attributes:: version Including git info if dev=True 0.5 or 0.5dev-r21-123acf1 - git_version Always including git info 0.5dev-r21-123acf1 - semantic_version Never including git info 0.5dev git_sha Git 7-digit SHA tag 123acf1 git_revs Git revision count 21 @@ -29,100 +27,26 @@ """ import os - -############################ -# SET THESE VALUES -############################ -# Major, Minor, Bugfix, Dev -VERSION = (4, 19, None, False) +import ska_helpers class SemanticVersion(object): - def __init__(self, major=0, minor=None, bugfix=None, dev=False): + def __init__(self, package): """ Semantic version object with support for git revisions - :param major: Major version - :param minor: Minor version - :param bugfix: Bugfix version - :param dev: True if this is a development version - :param version_filename: File name for package version.py + :param package: Package name """ - self.major = major - self.minor = minor - self.bugfix = bugfix - self.dev = dev + self.version = ska_helpers.get_version(package) + info = ska_helpers.version.parse_version(self.version) + self.major = info['major'] + self.minor = info['minor'] + self.bugfix = info['patch'] + self.dev = info['date'] is not None + self.git_revs = info['distance'] + self.git_sha = info['hash'] self.version_dir = os.path.abspath(os.path.dirname(__file__)) - def _get_git_info(self): - """ - Determines the number of revisions in this repository and returns "" if - this is not possible. - """ - try: - # First try getting git version from self.version_dir/GIT_VERSION - git_version_filename = os.path.join(self.version_dir, 'GIT_VERSION') - with open(git_version_filename, 'r') as fh: - git_revs, git_sha = fh.read().strip().split() - git_revs = int(git_revs) - - except Exception: - from subprocess import Popen, PIPE - try: - p = Popen(['git', 'rev-list', 'HEAD'], cwd=self.version_dir, - stdout=PIPE, stderr=PIPE, stdin=PIPE) - stdout, stderr = p.communicate() - - if p.returncode == 0: - revs = stdout.split('\n') - git_revs, git_sha = len(revs), revs[0][:7] - else: - git_revs, git_sha = None, None - except Exception: - git_revs, git_sha = None, None - - return git_revs, git_sha - - @property - def git_revs(self): - if not hasattr(self, '_git_revs'): - self._git_revs, self._git_sha = self._get_git_info() - return self._git_revs - - @property - def git_sha(self): - if not hasattr(self, '_git_sha'): - self._git_revs, self._git_sha = self._get_git_info() - return self._git_sha - - @property - def semantic_version(self): - _version = '{}'.format(self.major) - if self.minor is not None: - _version += '.{}'.format(self.minor) - if self.bugfix is not None: - _version += '.{}'.format(self.bugfix) - if self.dev: - _version += 'dev' - return _version - - @property - def git_version(self): - """ - Get the full version with git hashtag and release, e.g. - 0.5dev-r190-423abc1 - """ - if not hasattr(self, '_git_version'): - self._git_version = self.semantic_version - if self.git_revs and self.git_sha: - self._git_version += '-r{0}-{1}'.format(self.git_revs, self.git_sha) - - return self._git_version - - @property - def version(self): - return self.git_version if self.dev else self.semantic_version - def write_git_version_file(self): """ Make the full version with git hashtag and release from GIT_VERSION, @@ -134,18 +58,17 @@ def write_git_version_file(self): if os.path.exists(git_version_filename): os.unlink(git_version_filename) - # Remove existing attributes - for attr in ('_git_revs', '_git_sha'): - if hasattr(self, attr): - delattr(self, attr) - git_revs = self.git_revs git_sha = self.git_sha with open(git_version_filename, 'w') as fh: fh.write('{} {}'.format(git_revs, git_sha)) -package_version = SemanticVersion(*VERSION) +package_version = SemanticVersion(__package__) + __version__ = package_version.version -__git_version__ = package_version.git_version +__git_version__ = package_version.version +VERSION = (package_version.major, package_version.minor, package_version.bugfix, + package_version.dev) version = __version__ # For back-compatibility with legacy version.py + diff --git a/setup.py b/setup.py index e0938023..e72955dc 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ 'kadi_update_cmds = kadi.update_cmds:main']} setup(name='kadi', - version=package_version.version, + use_scm_version=True, + setup_requires = ['setuptools_scm', 'setuptools_scm_git_archive'], description='Kadi events archive', author='Tom Aldcroft', author_email='taldcroft@cfa.harvard.edu', From cbd1657128585c620d5d9e774b8851bb60e867ef Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Mon, 16 Dec 2019 16:26:41 -0500 Subject: [PATCH 02/35] flake8 --- kadi/version.py | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/kadi/version.py b/kadi/version.py index 17af2df5..69529cb8 100644 --- a/kadi/version.py +++ b/kadi/version.py @@ -71,4 +71,3 @@ def write_git_version_file(self): VERSION = (package_version.major, package_version.minor, package_version.bugfix, package_version.dev) version = __version__ # For back-compatibility with legacy version.py - diff --git a/setup.py b/setup.py index e72955dc..0c45261b 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup(name='kadi', use_scm_version=True, - setup_requires = ['setuptools_scm', 'setuptools_scm_git_archive'], + setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], description='Kadi events archive', author='Tom Aldcroft', author_email='taldcroft@cfa.harvard.edu', From f4df25e4a5b82151afeb9a4b6e286716e03871d2 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Tue, 17 Dec 2019 12:57:36 -0500 Subject: [PATCH 03/35] Removed version.py file to use setuptools_scm --- kadi/__init__.py | 4 ++- kadi/events/views.py | 4 +-- kadi/update_events.py | 6 ++-- kadi/version.py | 73 ------------------------------------------- setup.py | 7 ----- 5 files changed, 8 insertions(+), 86 deletions(-) delete mode 100644 kadi/version.py diff --git a/kadi/__init__.py b/kadi/__init__.py index 5fb05fe7..03b5eab0 100644 --- a/kadi/__init__.py +++ b/kadi/__init__.py @@ -1,6 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from .version import __version__, __git_version__ # noqa +import ska_helpers + +__version__ = ska_helpers.get_version(__package__) def test(*args, **kwargs): ''' diff --git a/kadi/events/views.py b/kadi/events/views.py index 5423fdcb..20676096 100644 --- a/kadi/events/views.py +++ b/kadi/events/views.py @@ -7,7 +7,7 @@ # Create your views here. from django.views.generic import ListView, TemplateView, DetailView from . import models -from ..version import __git_version__ +from .. import __version__ # Provide translation from event model class names like DarkCal to the URL name like dark_cal MODEL_NAMES = {m_class.__name__: m_name @@ -30,7 +30,7 @@ def get_sort(self): def get_context_data(self, **kwargs): # Call the base implementation first to get a context context = super(BaseView, self).get_context_data(**kwargs) - context['kadi_version'] = __git_version__ + context['kadi_version'] = __version__ event_models = models.get_event_models() # Make a list of tuples [(description1, name1), (description2, name2), ...] diff --git a/kadi/update_events.py b/kadi/update_events.py index 0cfdd955..e5861af7 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -203,10 +203,10 @@ def main(): os.environ['KADI'] = os.path.abspath(opt.data_root) from .paths import EVENTS_DB_PATH - from . import version + from . import __version__ from pprint import pformat - logger.info('Kadi version : {}'.format(version.__version__)) - logger.info('Kadi path : {}'.format(os.path.dirname(os.path.abspath(version.__file__)))) + logger.info('Kadi version : {}'.format(__version__)) + logger.info('Kadi path : {}'.format(os.path.dirname(os.path.abspath(__file__)))) logger.info('Event database : {}'.format(EVENTS_DB_PATH())) logger.info('') logger.info('Options:') diff --git a/kadi/version.py b/kadi/version.py deleted file mode 100644 index 69529cb8..00000000 --- a/kadi/version.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -Version numbering template. - -This file is intended to be copied into a project and provide version information, -including git version values. It is copied instead of used as a module to ensure -that projects can always run their setup.py with no external dependency. - -The `major`, `minor`, and `bugfix` variables hold the respective parts -of the version number. Basic usage is to copy this file into your package directory -alongside __init__.py. Then use as follows:: - - from package.version import version_object - from package.version import __version__ # version_object.version - -In addition to ``major``, ``minor``, ``bugfix``, and ``dev`` attributes, the object -``version_object`` has a number of useful attributes:: - - version Including git info if dev=True 0.5 or 0.5dev-r21-123acf1 - git_sha Git 7-digit SHA tag 123acf1 - git_revs Git revision count 21 - -See ``setup.py`` in the ``versioning`` package for an example of what needs to -be done there. The key step is ``version_object.write_git_version_file()`` -in order to create a package file ``git_version.py`` alongside ``version.py`` -which retains the git information outside of the git repo. -""" - -import os -import ska_helpers - - -class SemanticVersion(object): - def __init__(self, package): - """ - Semantic version object with support for git revisions - - :param package: Package name - """ - self.version = ska_helpers.get_version(package) - info = ska_helpers.version.parse_version(self.version) - self.major = info['major'] - self.minor = info['minor'] - self.bugfix = info['patch'] - self.dev = info['date'] is not None - self.git_revs = info['distance'] - self.git_sha = info['hash'] - self.version_dir = os.path.abspath(os.path.dirname(__file__)) - - def write_git_version_file(self): - """ - Make the full version with git hashtag and release from GIT_VERSION, - typically during `python setup.py sdist` - """ - git_version_filename = os.path.join(self.version_dir, 'GIT_VERSION') - - # Remove any existing GIT_VERSION file - if os.path.exists(git_version_filename): - os.unlink(git_version_filename) - - git_revs = self.git_revs - git_sha = self.git_sha - with open(git_version_filename, 'w') as fh: - fh.write('{} {}'.format(git_revs, git_sha)) - - -package_version = SemanticVersion(__package__) - -__version__ = package_version.version -__git_version__ = package_version.version -VERSION = (package_version.major, package_version.minor, package_version.bugfix, - package_version.dev) -version = __version__ # For back-compatibility with legacy version.py diff --git a/setup.py b/setup.py index 0c45261b..b999577e 100644 --- a/setup.py +++ b/setup.py @@ -3,13 +3,6 @@ import os import sys -# from this_package.version import package_version object -from kadi.version import package_version - -# Write GIT revisions and SHA tag into -# (same directory as version.py) -package_version.write_git_version_file() - foundation_files = ['static/foundation/css/*', 'static/foundation/js/foundation/*', 'static/foundation/js/vendor/*', From a0ca23dd0f87a667d39543133793efcdf5957364 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Tue, 17 Dec 2019 13:28:47 -0500 Subject: [PATCH 04/35] flake8 --- kadi/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kadi/__init__.py b/kadi/__init__.py index 03b5eab0..c7b81191 100644 --- a/kadi/__init__.py +++ b/kadi/__init__.py @@ -4,6 +4,7 @@ __version__ = ska_helpers.get_version(__package__) + def test(*args, **kwargs): ''' Run py.test unit tests. From f574a57a7f4fa648b05b7475c17299a3cd899c49 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Tue, 17 Dec 2019 15:35:19 -0500 Subject: [PATCH 05/35] found and removed uses of version.py --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 3a266110..cc85dd68 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,7 +62,7 @@ # built documents. # # The full version, including alpha/beta/rc tags. -from kadi.version import version as release +from kadi import __version__ as release # The short X.Y version. version = '.'.join(release.split('.')[:2]) From 703ea790bd0b3596be5c3df13da92c8578979770 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Thu, 19 Dec 2019 12:49:00 -0500 Subject: [PATCH 06/35] for setuptools_scm to work, use setuptools.setup and not distutils.core.setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b999577e..786a7fff 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from distutils.core import setup +from setuptools import setup import os import sys From 5a64bb5192e0d0237a36d6512e5c7ee68986e915 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Aug 2019 11:55:00 -0400 Subject: [PATCH 07/35] Set env vars to work on new web-kadi / web-kadi-test servers --- kadi/wsgi.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kadi/wsgi.py b/kadi/wsgi.py index 5a248ebc..872862cb 100644 --- a/kadi/wsgi.py +++ b/kadi/wsgi.py @@ -18,6 +18,14 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kadi.settings") +# When running production web site there should be no HOME, but +# some packages (e.g. sherpa) assume its existence. +os.environ.setdefault("HOME", "/tmp") + +# Need SYBASE for some apps like star history. +# TODO: remove dependence on sybase, use mica guide / acq stats instead. +os.environ.setdefault("SYBASE", "/soft/SYBASE15.7") + # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. From a721dab9f35bf13cd9770f75eb1ad0822e0fb0c1 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 9 Oct 2019 08:54:12 -0400 Subject: [PATCH 08/35] Initial Py3 / django 2.2 fixes --- kadi/events/models.py | 9 +++++---- kadi/events/urls.py | 8 ++++---- kadi/urls.py | 24 ++++++++++++------------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index fb51c483..f49f484e 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -1602,7 +1602,8 @@ class Dwell(Event): ============ ============ ======================================== """ rel_tstart = models.FloatField(help_text='Start time relative to manvr end (sec)') - manvr = models.ForeignKey(Manvr, help_text='Maneuver that contains this dwell') + manvr = models.ForeignKey(Manvr, on_delete=models.CASCADE, + help_text='Maneuver that contains this dwell') ra = models.FloatField(help_text='Right ascension (deg)') dec = models.FloatField(help_text='Declination (deg)') roll = models.FloatField(help_text='Roll angle (deg)') @@ -1643,7 +1644,7 @@ class ManvrSeq(BaseModel): =========== ============ =========== """ - manvr = models.ForeignKey(Manvr) + manvr = models.ForeignKey(Manvr, on_delete=models.CASCADE) msid = models.CharField(max_length=8) prev_val = models.CharField(max_length=4) val = models.CharField(max_length=4) @@ -2249,7 +2250,7 @@ class OrbitPoint(BaseModel): descr Char(50) =========== ============ =========== """ - orbit = models.ForeignKey(Orbit) + orbit = models.ForeignKey(Orbit, on_delete=models.CASCADE) date = models.CharField(max_length=21) name = models.CharField(max_length=9) orbit_num = models.IntegerField() @@ -2282,7 +2283,7 @@ class RadZone(Event): perigee Char(21) =========== ============ ================================ """ - orbit = models.ForeignKey(Orbit) + orbit = models.ForeignKey(Orbit, on_delete=models.CASCADE) orbit_num = models.IntegerField() perigee = models.CharField(max_length=21) diff --git a/kadi/events/urls.py b/kadi/events/urls.py index e1349bdc..63b1da59 100644 --- a/kadi/events/urls.py +++ b/kadi/events/urls.py @@ -1,7 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import inspect -from django.conf.urls import patterns, url +from django.conf.urls import url from . import views @@ -31,6 +31,6 @@ except KeyError: pass -urlpatterns = patterns('', - url('^$', views.IndexView.as_view()), - *urls) +urlpatterns = [# '', + url('^$', views.IndexView.as_view()), + *urls] diff --git a/kadi/urls.py b/kadi/urls.py index 569e5dce..0bc62b5d 100644 --- a/kadi/urls.py +++ b/kadi/urls.py @@ -1,23 +1,23 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from django.conf.urls import patterns, include, url +from django.conf.urls import include, url from django.contrib import admin import kadi.views import mica.web.views -import find_attitude.web.views +# import find_attitude.web.views admin.autodiscover() -urlpatterns = patterns('', - url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', include(admin.site.urls)), - url(r'^kadi/events/', include('kadi.events.urls')), - url(r'^$', kadi.views.IndexView.as_view()), - url(r'^mica/$', mica.web.views.IndexView.as_view()), - url(r'^pcad_acq/$', mica.web.views.AcqView.as_view()), - url(r'^star_hist/$', mica.web.views.StarHistView.as_view()), - url(r'^find_attitude/$', find_attitude.web.views.index), - ) +urlpatterns = [ # '', + url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + url(r'^admin/', include(admin.site.urls)), + url(r'^kadi/events/', include('kadi.events.urls')), + url(r'^$', kadi.views.IndexView.as_view()), + url(r'^mica/$', mica.web.views.IndexView.as_view()), + url(r'^pcad_acq/$', mica.web.views.AcqView.as_view()), + url(r'^star_hist/$', mica.web.views.StarHistView.as_view()), + # url(r'^find_attitude/$', find_attitude.web.views.index), +] # Another way to do this, corresponds to commented code in kadi/views.py # urlpatterns = patterns('', From 3c5ad962ab1409a9f4e3b350ac2615fe77489e66 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 9 Oct 2019 09:56:40 -0400 Subject: [PATCH 09/35] More fixes --- kadi/events/__init__.py | 5 +++-- kadi/events/models.py | 3 ++- kadi/settings.py | 31 ++++++++++++++++++++++++------- kadi/urls.py | 15 +-------------- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/kadi/events/__init__.py b/kadi/events/__init__.py index 26598b39..509b21dc 100644 --- a/kadi/events/__init__.py +++ b/kadi/events/__init__.py @@ -51,9 +51,10 @@ # This attribute does not exist in 1.6. # https://docs.djangoproject.com/en/1.10/topics/settings/#calling-django-setup-is-required-for-standalone-django-usage try: - django.setup() + # django.setup() + pass except AttributeError: pass # from .models import * -from .query import * # noqa +# from .query import * # noqa diff --git a/kadi/events/models.py b/kadi/events/models.py index f49f484e..5c7c1fe4 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -2103,7 +2103,8 @@ class DsnComm(IFotEvent): oc = models.CharField(max_length=30, help_text='OC crew') cc = models.CharField(max_length=30, help_text='CC crew') pass_plan = models.OneToOneField(PassPlan, help_text='Pass plan', null=True, - related_name='dsn_comm') + related_name='dsn_comm', + on_delete=models.CASCADE) ifot_type_desc = 'DSN_COMM' ifot_props = ['bot', 'eot', 'activity', 'config', 'data_rate', 'site', 'soe', 'station'] diff --git a/kadi/settings.py b/kadi/settings.py index 5e3fbc02..cfce3762 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -115,7 +115,7 @@ INSTALLED_APPS += (app,) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -181,9 +181,26 @@ join(BASE_DIR, 'kadi/static'), ) -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. - join(BASE_DIR, 'kadi/templates'), -) +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + # insert your TEMPLATE_DIRS here + join(BASE_DIR, 'kadi/templates'), + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this + # list if you haven't customized them: + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] diff --git a/kadi/urls.py b/kadi/urls.py index 0bc62b5d..f0b78b9f 100644 --- a/kadi/urls.py +++ b/kadi/urls.py @@ -9,8 +9,7 @@ admin.autodiscover() urlpatterns = [ # '', - url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', include(admin.site.urls)), + # url(r'^admin/', include(admin.site.urls)), url(r'^kadi/events/', include('kadi.events.urls')), url(r'^$', kadi.views.IndexView.as_view()), url(r'^mica/$', mica.web.views.IndexView.as_view()), @@ -18,15 +17,3 @@ url(r'^star_hist/$', mica.web.views.StarHistView.as_view()), # url(r'^find_attitude/$', find_attitude.web.views.index), ] - -# Another way to do this, corresponds to commented code in kadi/views.py -# urlpatterns = patterns('', -# url(r'^admin/doc/', include('django.contrib.admindocs.urls')), -# url(r'^admin/', include(admin.site.urls)), -# url(r'^mica/$', mica.web.views.IndexView.as_view()), -# url(r'^kadi/events/', include('kadi.events.urls')), - -# # Eventually consider forcing the kadi page to be the default page if other -# # urls fail to match. For now keep out the ".*" for debug purposes. -# url(r'^$', kadi.views.index, name='index'), -# ) From 3d6fd0674a93d0bc32e6fb241faabf06bc51d810 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 9 Oct 2019 10:24:28 -0400 Subject: [PATCH 10/35] Make it work as server and as standalone --- kadi/events/__init__.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/kadi/events/__init__.py b/kadi/events/__init__.py index 509b21dc..e246ef22 100644 --- a/kadi/events/__init__.py +++ b/kadi/events/__init__.py @@ -44,17 +44,12 @@ import os import django +# If we are running standalone then this ENV var is not set. Need to explicitly +# do the setup and then import query module attributes for standalone. +# For WSGI server the env var is set in wsgi.py. +# For the dev server it is set in manage.py. + if 'DJANGO_SETTINGS_MODULE' not in os.environ: os.environ['DJANGO_SETTINGS_MODULE'] = 'kadi.settings' - -# In django >= 1.8 an explicit call to setup is required for standalone. -# This attribute does not exist in 1.6. -# https://docs.djangoproject.com/en/1.10/topics/settings/#calling-django-setup-is-required-for-standalone-django-usage -try: - # django.setup() - pass -except AttributeError: - pass - -# from .models import * -# from .query import * # noqa + django.setup() + from .query import * # noqa From 8b68dc9f5382f05b1d7946ef784561ed850fe3d2 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Dec 2019 10:05:33 -0500 Subject: [PATCH 11/35] Fixes for update_events with django 2.2 --- NOTES.build | 27 ++++----------------------- kadi/events/models.py | 1 + kadi/events/scrape.py | 13 +++++++------ kadi/update_events.py | 39 ++++++++++++++++++++++----------------- 4 files changed, 34 insertions(+), 46 deletions(-) diff --git a/NOTES.build b/NOTES.build index 52a6e96e..c1c2dc5b 100644 --- a/NOTES.build +++ b/NOTES.build @@ -5,36 +5,17 @@ cd ~/git/kadi export KADI=$PWD rm events.db3 cmds.h5 cmds.pkl -./manage.py syncdb +./manage.py makemigrations events +./manage.py migrate -export ENG_ARCHIVE=/proj/sot/ska/data/eng_archive/1999 ./update_events --start=1999:200 --stop=2000:001 --model=SafeSun - -export ENG_ARCHIVE= ./update_events --start=2000:001 ./update_cmds --start=2000:001 -NOTE: could potentially get all the 1999 events using the 1999 eng - archive. - ################################################################# -# Remove certain tables in a local version of the flight events database +# Re-build single table ################################################################# % export KADI=$PWD % cp /proj/sot/ska/data/kadi/events.db3 ./ -% sqlite3 events.db3 -> drop table events_cap; - -% ./manage.py syncdb -% ./update_events --start=2000:001 --model=CAP --delete-from-start - - -################################################################# -# Reprocess a table from a starting date in a local version of -# the flight events database -################################################################# - -% cp /proj/sot/ska/data/kadi/events.db3 ./ -% ./update_events --start=2012:001 --model=Scs107 --delete-from-start - +% ./update_events --start=1999:001 --model=CAP --delete-from-start diff --git a/kadi/events/models.py b/kadi/events/models.py index 5c7c1fe4..6b31d64d 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -1833,6 +1833,7 @@ def get_events(cls, start, stop=None): # Manually generate a unique key for event since date is not unique for event in events: key = ''.join(event[x] for x in ('start', 'descr', 'note', 'source')) + key = key.encode('ascii', 'replace') # Should already be clean ASCII but make sure event['key'] = event['start'] + ':' + hashlib.sha1(key).hexdigest()[:6] return events diff --git a/kadi/events/scrape.py b/kadi/events/scrape.py index 146c5c2c..dbf0711f 100644 --- a/kadi/events/scrape.py +++ b/kadi/events/scrape.py @@ -1,7 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import re -from BeautifulSoup import BeautifulSoup as parse_html +from bs4 import BeautifulSoup as parse_html from Chandra.Time import DateTime from .. import occweb @@ -23,8 +23,9 @@ def cleantext(text): """ Clean up some HTML or unicode special characters and replace with ASCII equivalents. """ - lines = text.encode('ascii', 'xmlcharrefreplace').splitlines() - lines = [x.strip() for x in lines] + # Convert any non-ASCII characters to XML like 'ꀀabcd޴' + text = text.encode('ascii', 'xmlcharrefreplace').decode('ascii') + lines = [x.strip() for x in text.splitlines()] text = ' '.join(lines) for match, out in REPLACES: text = re.sub(match, out, text) @@ -41,7 +42,7 @@ def get_table_rows(table): trs = table.findAll('tr') for tr in trs: tds = tr.findAll('td') - if u'colspan' in dict(tds[0].attrs): + if 'colspan' in dict(tds[0].attrs): continue vals = tuple(cleantext(td.text) for td in tds) rows.append(vals) @@ -63,7 +64,7 @@ def get_fdb_major_events(): return evts_cache[page] html = occweb.get_url(page) - soup = parse_html(html) + soup = parse_html(html, 'lxml') table = soup.find('table') rows = get_table_rows(table) evts = [] @@ -107,7 +108,7 @@ def get_fot_major_events(): return evts_cache[page] html = occweb.get_url(page) - soup = parse_html(html) + soup = parse_html(html, 'lxml') table = soup.find('table', attrs={'class': 'MsoNormalTable'}) rows = get_table_rows(table) evts = [] diff --git a/kadi/update_events.py b/kadi/update_events.py index e5861af7..ba346d95 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -150,25 +150,30 @@ def update(EventModel, date_stop): .format(cls_name, date_start.date[:-4], date_stop.date[:-4])) # Get events for this model from telemetry. This is returned as a list - # of dicts with key/val pairs corresponding to model fields - events = EventModel.get_events(date_start, date_stop) - - with django.db.transaction.commit_on_success(): - for event in events: - # Try to save event. Use force_insert=True because otherwise django will - # update if the event primary key already exists. In this case we want to - # force an exception and move on to the next event. - try: - event_model = EventModel.from_dict(event, logger) - try4times(event_model.save, force_insert=True) - except django.db.utils.IntegrityError as err: - if not re.search('unique', str(err), re.IGNORECASE): - raise - logger.verbose('Skipping {} at {}: already in database ({})' - .format(cls_name, event['start'], err)) - continue + # of dicts with key/val pairs corresponding to model fields. + events_in_dates = EventModel.get_events(date_start, date_stop) + + # Determine which of the events is not already in the database and + # put them in a list for saving. + events = [] + event_models = [] + for event in events_in_dates: + event_model = EventModel.from_dict(event, logger) + try: + EventModel.objects.get(pk=event_model.pk) + except EventModel.DoesNotExist: + events.append(event) + event_models.append(event_model) + else: + logger.verbose('Skipping {} at {}: already in database' + .format(cls_name, event['start'])) + # Save the new events in an atomic fashion + with django.db.transaction.atomic(): + for event, event_model in zip(events, event_models): + try4times(event_model.save) logger.info('Added {} {}'.format(cls_name, event_model)) + if 'dur' in event and event['dur'] < 0: logger.info('WARNING: negative event duration for {} {}' .format(cls_name, event_model)) From 436002ddfbd91ff23be47137716c229a42cc3026 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Dec 2019 11:18:36 -0500 Subject: [PATCH 12/35] Make obsid a generic field of TlmEvent --- kadi/events/models.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index 6b31d64d..5e669385 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -411,6 +411,19 @@ def from_dict(cls, model_dict, logger=None): Model. If `logger` is supplied then log output at debug level. """ model = cls() + + # Get the obsid at the appropriate time for the event (typically "start" + # but "stop" in the case of Manvr). + if isinstance(model, TlmEvent): + tstart = DateTime(model_dict[cls._get_obsid_start_attr]).secs + obsrq = fetch.Msid('cobsrqid', tstart, tstart + 200) + if len(obsrq.vals) == 0: + logger.warn(f'unable to get COBSRQID near {model_dict["datestart"]}, ' + f'using obsid=-999') + model_dict['obsid'] = -999 + else: + model_dict['obsid'] = obsrq.vals[0] + for key, val in model_dict.items(): if hasattr(model, key): if logger is not None: @@ -585,6 +598,8 @@ def __unicode__(self): class TlmEvent(Event): + obsid = models.IntegerField(help_text='Observation ID (COBSRQID)') + event_msids = None # must be overridden by derived class event_val = None event_filter_bad = True # Normally remove bad quality data immediately @@ -757,8 +772,6 @@ class Obsid(TlmEvent): """ event_msids = ['cobsrqid'] - obsid = models.IntegerField(help_text='Observation ID (COBSRQID)') - update_priority = 1000 # Process Obsid first @classmethod From a9a6323a3520df63379d1d6e2cb4e67ed0338118 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Dec 2019 11:39:34 -0500 Subject: [PATCH 13/35] Get rid of the local ./update_* launcher scripts --- NOTES.add_event_type | 6 +++--- NOTES.build | 8 ++++---- NOTES.reprocess | 2 +- NOTES.update_ltt_bads | 4 ++-- update_cmds | 9 --------- update_events | 12 ------------ 6 files changed, 10 insertions(+), 31 deletions(-) delete mode 100755 update_cmds delete mode 100755 update_events diff --git a/NOTES.add_event_type b/NOTES.add_event_type index 33557875..d3e79b1b 100644 --- a/NOTES.add_event_type +++ b/NOTES.add_event_type @@ -34,14 +34,14 @@ export KADI=$PWD ./manage.py syncdb -./update_events --start=1999:200 --stop=2001:001 --model=${ModelClassName} +python -m kadi.update_events --start=1999:200 --stop=2001:001 --model=${ModelClassName} # Update early events first and look for warnings. Also confirm that the first event # matches what is in the current database unless a change is intended. # Some events may need a later start data to be fully sampled. Manvr, for example, # should use a start of "1999:251" (1999:230 + 21 days lookback). # Probably not needed for events that rely on only one event MSID. -./update_events --start=2001:001 --model=${ModelClassName} +python -m kadi.update_events --start=2001:001 --model=${ModelClassName} # Update the rest of the time range for the updated model/event @@ -104,7 +104,7 @@ Use the current flight kadi code and the test database and confirm that kadi tes In [9]: kadi.test() ============================= test session starts ============================== platform linux2 -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4 - collected 15 items + collected 15 items kadi/tests/test_events.py .......... kadi/tests/test_occweb.py ..... diff --git a/NOTES.build b/NOTES.build index c1c2dc5b..812ccd0a 100644 --- a/NOTES.build +++ b/NOTES.build @@ -8,9 +8,9 @@ rm events.db3 cmds.h5 cmds.pkl ./manage.py makemigrations events ./manage.py migrate -./update_events --start=1999:200 --stop=2000:001 --model=SafeSun -./update_events --start=2000:001 -./update_cmds --start=2000:001 +python -m kadi.update_events --start=1999:200 --stop=2000:001 --model=SafeSun +python -m kadi.update_events --start=2000:001 +python -m kadi.update_cmds --start=2000:001 ################################################################# # Re-build single table @@ -18,4 +18,4 @@ rm events.db3 cmds.h5 cmds.pkl % export KADI=$PWD % cp /proj/sot/ska/data/kadi/events.db3 ./ -% ./update_events --start=1999:001 --model=CAP --delete-from-start +% python -m kadi.update_events --start=1999:001 --model=CAP --delete-from-start diff --git a/NOTES.reprocess b/NOTES.reprocess index 28d4f361..4ac6755c 100644 --- a/NOTES.reprocess +++ b/NOTES.reprocess @@ -11,6 +11,6 @@ If the kadi events.db3 database gets corrupted (e.g. problems related to the cp /proj/sot/ska/data/kadi/events.db3 ./ # Update from 2017:017 (for example) to present - ./update_events --start=2017:017 --delete-from-start + python -m kadi.update_events --start=2017:017 --delete-from-start cp events.db3 /proj/sot/ska/data/kadi/ diff --git a/NOTES.update_ltt_bads b/NOTES.update_ltt_bads index 239f8414..4a87576a 100644 --- a/NOTES.update_ltt_bads +++ b/NOTES.update_ltt_bads @@ -8,7 +8,7 @@ Test ---- export KADI=$PWD -./update_events --start '1999:100' --delete-from-start --loop-days=10000 \ +python -m kadi.update_events --start '1999:100' --delete-from-start --loop-days=10000 \ --model=LttBad >& run.log & # Confirm new events added as expected @@ -23,7 +23,7 @@ make install # or just copy ltt_bads.dat to $ska/data/kadi/. unset KADI cd $ska/share/kadi -./update_events --start '1999:100' --delete-from-start --loop-days=10000 \ +python -m kadi.update_events --start '1999:100' --delete-from-start --loop-days=10000 \ --data-root=$ska/data/kadi --model=LttBad >& run.log & diff --git a/update_cmds b/update_cmds deleted file mode 100755 index 5faa7edd..00000000 --- a/update_cmds +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python -""" -Update the cmd_states table to reflect current load segments / timelines -in database. This is normally run as a cron job. -""" - -if __name__ == '__main__': - from kadi import update_cmds - update_cmds.main() diff --git a/update_events b/update_events deleted file mode 100755 index 21006d3a..00000000 --- a/update_events +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python -""" -Update the cmd_states table to reflect current load segments / timelines -in database. This is normally run as a cron job. -""" - -import os - -if __name__ == '__main__': - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kadi.settings") - from kadi import update_events - update_events.main() From 25d655ce444a7e4e1878d1e3f4334dc5717369e7 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Dec 2019 11:56:05 -0500 Subject: [PATCH 14/35] Do not override obsid field in Obsid model --- kadi/events/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index 5e669385..582d9906 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -414,7 +414,7 @@ def from_dict(cls, model_dict, logger=None): # Get the obsid at the appropriate time for the event (typically "start" # but "stop" in the case of Manvr). - if isinstance(model, TlmEvent): + if isinstance(model, TlmEvent) and model is not Obsid: tstart = DateTime(model_dict[cls._get_obsid_start_attr]).secs obsrq = fetch.Msid('cobsrqid', tstart, tstart + 200) if len(obsrq.vals) == 0: From 9eadc8e9fa2f7d8f8cd2a6cef9c2dcd0890992a0 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Dec 2019 18:35:08 -0500 Subject: [PATCH 15/35] More changes for successfully populating database --- kadi/events/models.py | 5 ++--- kadi/events/orbit_funcs.py | 26 ++++++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index 582d9906..779f832f 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -418,7 +418,7 @@ def from_dict(cls, model_dict, logger=None): tstart = DateTime(model_dict[cls._get_obsid_start_attr]).secs obsrq = fetch.Msid('cobsrqid', tstart, tstart + 200) if len(obsrq.vals) == 0: - logger.warn(f'unable to get COBSRQID near {model_dict["datestart"]}, ' + logger.warn(f'unable to get COBSRQID near {model_dict[cls._get_obsid_start_attr]}, ' f'using obsid=-999') model_dict['obsid'] = -999 else: @@ -2404,8 +2404,7 @@ class LttBad(AsciiTableEvent): intervals_file = 'ltt_bads.dat' # Table.read keyword args - table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', guess=False, - fill_values=None) + table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', guess=False) start_column = 'start' stop_column = 'stop' diff --git a/kadi/events/orbit_funcs.py b/kadi/events/orbit_funcs.py index fa23221c..d1cd0d61 100644 --- a/kadi/events/orbit_funcs.py +++ b/kadi/events/orbit_funcs.py @@ -14,14 +14,14 @@ class NotFoundError(Exception): pass -ORBIT_POINTS_DTYPE = [('date', 'S21'), ('name', 'S8'), - ('orbit_num', 'i4'), ('descr', 'S50')] +ORBIT_POINTS_DTYPE = [('date', 'U21'), ('name', 'U8'), + ('orbit_num', 'i4'), ('descr', 'U50')] ORBITS_DTYPE = [('orbit_num', 'i4'), - ('start', 'S21'), ('stop', 'S21'), + ('start', 'U21'), ('stop', 'U21'), ('tstart', 'f8'), ('tstop', 'f8'), ('dur', 'f4'), - ('perigee', 'S21'), ('t_perigee', 'f8'), ('apogee', 'S21'), - ('start_radzone', 'S21'), ('stop_radzone', 'S21'), + ('perigee', 'U21'), ('t_perigee', 'f8'), ('apogee', 'U21'), + ('start_radzone', 'U21'), ('stop_radzone', 'U21'), ('dt_start_radzone', 'f4'), ('dt_stop_radzone', 'f4')] logger = logging.getLogger('events') @@ -235,6 +235,9 @@ def interpolate_orbit_points(orbit_points, name): """ Linearly interpolate across any gaps for ``name`` orbit_points. """ + if len(orbit_points) == 0: + return [] + ok = orbit_points['name'] == name ops = orbit_points[ok] # Get the indexes of missing orbits @@ -267,12 +270,15 @@ def process_orbit_points(orbit_points): Returns a numpy array with processed orbit points:: - ORBIT_POINTS_DTYPE = [('date', 'S21'), ('name', 'S8'), - ('orbit_num', 'i4'), ('descr', 'S50')] + ORBIT_POINTS_DTYPE = [('date', 'U21'), ('name', 'U8'), + ('orbit_num', 'i4'), ('descr', 'U50')] """ # Find neighboring pairs of orbit points that are identical except for date. # If the dates are then within 180 seconds of each other then toss the first # of the pair. + if len(orbit_points) == 0: + return np.array([], dtype=ORBIT_POINTS_DTYPE) + uniq_orbit_points = [] for op0, op1 in zip(orbit_points[:-1], orbit_points[1:]): if op0[1:4] == op1[1:4]: @@ -345,10 +351,10 @@ def get_orbits(orbit_points): Returns a numpy structured array:: ORBITS_DTYPE = [('orbit_num', 'i4'), - ('start', 'S21'), ('stop', 'S21'), + ('start', 'U21'), ('stop', 'U21'), ('tstart', 'f8'), ('tstop', 'f8'), ('dur', 'f4'), - ('perigee', 'S21'), ('t_perigee', 'f8'), ('apogee', 'S21'), - ('start_radzone', 'S21'), ('stop_radzone', 'S21'), + ('perigee', 'U21'), ('t_perigee', 'f8'), ('apogee', 'U21'), + ('start_radzone', 'U21'), ('stop_radzone', 'U21'), ('dt_start_radzone', 'f4'), ('dt_stop_radzone', 'f4')] """ def get_idx(ops, name): From 5fceb681460da5423eca4c7e0b85d62251b4838a Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Dec 2019 08:14:42 -0500 Subject: [PATCH 16/35] Improve fault tolerance --- kadi/update_events.py | 89 ++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/kadi/update_events.py b/kadi/update_events.py index ba346d95..d771effe 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -153,6 +153,52 @@ def update(EventModel, date_stop): # of dicts with key/val pairs corresponding to model fields. events_in_dates = EventModel.get_events(date_start, date_stop) + # Determine which of the events is not already in the database and + # put them in a list for saving. + event_models, events = get_events_and_event_models(EventModel, cls_name, events_in_dates) + + # Save the new events in an atomic fashion + with django.db.transaction.atomic(): + for event, event_model in zip(events, event_models): + try: + # In order to catch an IntegrityError here and press on, need to + # wrap this in atomic(). This was driven by bad data in iFOT, namely + # duplicate PassPlans that point to the same DsnComm, which gives an + # IntegrityError because those are related as one-to-one. + with django.db.transaction.atomic(): + save_event_to_database(cls_name, event, event_model, models) + except django.db.utils.IntegrityError: + import traceback + logger.warn(f'WARNING: IntegrityError skipping {event_model}') + logger.warn(f'Event dict:\n{event}') + logger.warn(f'Traceback:\n{traceback.format_exc()}') + continue + + # If processing got here with no exceptions then save the event update + # information to database + update.save() + + +def save_event_to_database(cls_name, event, event_model, models): + try4times(event_model.save) + logger.info('Added {} {}'.format(cls_name, event_model)) + if 'dur' in event and event['dur'] < 0: + logger.info('WARNING: negative event duration for {} {}' + .format(cls_name, event_model)) + # Add any foreign rows (many to one) + for foreign_cls_name, rows in event.get('foreign', {}).items(): + ForeignModel = getattr(models, foreign_cls_name) + if isinstance(rows, np.ndarray): + rows = [{key: row[key].tolist() for key in row.dtype.names} for row in rows] + for row in rows: + # Convert to a plain dict if row is structured array + foreign_model = ForeignModel.from_dict(row, logger) + setattr(foreign_model, event_model.model_name, event_model) + logger.verbose('Adding {}'.format(foreign_model)) + try4times(foreign_model.save) + + +def get_events_and_event_models(EventModel, cls_name, events_in_dates): # Determine which of the events is not already in the database and # put them in a list for saving. events = [] @@ -167,32 +213,7 @@ def update(EventModel, date_stop): else: logger.verbose('Skipping {} at {}: already in database' .format(cls_name, event['start'])) - - # Save the new events in an atomic fashion - with django.db.transaction.atomic(): - for event, event_model in zip(events, event_models): - try4times(event_model.save) - logger.info('Added {} {}'.format(cls_name, event_model)) - - if 'dur' in event and event['dur'] < 0: - logger.info('WARNING: negative event duration for {} {}' - .format(cls_name, event_model)) - - # Add any foreign rows (many to one) - for foreign_cls_name, rows in event.get('foreign', {}).items(): - ForeignModel = getattr(models, foreign_cls_name) - if isinstance(rows, np.ndarray): - rows = [{key: row[key].tolist() for key in row.dtype.names} for row in rows] - for row in rows: - # Convert to a plain dict if row is structured array - foreign_model = ForeignModel.from_dict(row, logger) - setattr(foreign_model, event_model.model_name, event_model) - logger.verbose('Adding {}'.format(foreign_model)) - try4times(foreign_model.save) - - # If processing got here with no exceptions then save the event update - # information to database - update.save() + return event_models, events def main(): @@ -254,11 +275,17 @@ def main(): EventModels = sorted(EventModels, key=lambda x: x.update_priority, reverse=True) for EventModel in EventModels: - if opt.delete_from_start and opt.start is not None: - delete_from_date(EventModel, opt.start) - - for date_stop in date_stops: - update(EventModel, date_stop) + try: + if opt.delete_from_start and opt.start is not None: + delete_from_date(EventModel, opt.start) + + for date_stop in date_stops: + update(EventModel, date_stop) + except Exception: + # Something went wrong, but press on with processing other EventModels + import traceback + logger.error(f'ERROR in processing {EventModel}') + logger.error(f'Traceback:\n{traceback.format_exc()}') if opt.ftp: # Push events database file to OCC via lucky ftp From e417fa9555c8304c0938ed50a2f0f6ce6a315436 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Dec 2019 08:39:32 -0500 Subject: [PATCH 17/35] Remove six and PY2 compatibility --- kadi/cmds/cmds.py | 10 ++++------ kadi/commands/commands.py | 19 ++++++------------- kadi/commands/states.py | 6 ++---- kadi/events/json_field.py | 6 ++---- kadi/events/models.py | 21 ++++++--------------- kadi/events/query.py | 8 +------- kadi/events/views.py | 2 +- kadi/occweb.py | 3 +-- kadi/settings.py | 2 +- kadi/update_events.py | 3 +-- 10 files changed, 25 insertions(+), 55 deletions(-) diff --git a/kadi/cmds/cmds.py b/kadi/cmds/cmds.py index f5a09fb9..62d12954 100644 --- a/kadi/cmds/cmds.py +++ b/kadi/cmds/cmds.py @@ -5,8 +5,7 @@ from astropy.table import Table from Chandra.Time import DateTime -import six -from six.moves import cPickle as pickle +import pickle from ..paths import IDX_CMDS_PATH, PARS_DICT_PATH @@ -51,8 +50,7 @@ def load_idx_cmds(): def load_pars_dict(): with open(PARS_DICT_PATH(), 'rb') as fh: - kwargs = {} if six.PY2 else {'encoding': 'ascii'} - pars_dict = pickle.load(fh, **kwargs) + pars_dict = pickle.load(fh, encoding='ascii') return pars_dict @@ -141,7 +139,7 @@ def _find(start=None, stop=None, **kwargs): ok &= idx_cmds['date'] < DateTime(stop).date for key, val in kwargs.items(): key = key.lower() - if isinstance(val, six.string_types): + if isinstance(val, str): val = val.upper() if key in idx_cmds.dtype.names: ok &= idx_cmds[key] == val @@ -203,7 +201,7 @@ def __len__(self): def __getitem__(self, item): cmds = self.cmds - if isinstance(item, six.string_types): + if isinstance(item, str): if item in cmds.colnames: return cmds[item] diff --git a/kadi/commands/commands.py b/kadi/commands/commands.py index baf8a3a3..25a1be89 100644 --- a/kadi/commands/commands.py +++ b/kadi/commands/commands.py @@ -6,8 +6,7 @@ from astropy.table import Table, Row, Column, vstack from Chandra.Time import DateTime -import six -from six.moves import cPickle as pickle +import pickle from ..paths import IDX_CMDS_PATH, PARS_DICT_PATH @@ -52,8 +51,7 @@ def load_idx_cmds(): def load_pars_dict(): with open(PARS_DICT_PATH(), 'rb') as fh: - kwargs = {} if six.PY2 else {'encoding': 'ascii'} - pars_dict = pickle.load(fh, **kwargs) + pars_dict = pickle.load(fh, encoding='ascii') return pars_dict @@ -195,7 +193,7 @@ def _find(start=None, stop=None, **kwargs): ok &= idx_cmds['date'] < DateTime(stop).date for key, val in kwargs.items(): key = key.lower() - if isinstance(val, six.string_types): + if isinstance(val, str): val = val.upper() if key in idx_cmds.dtype.names: ok &= idx_cmds[key] == val @@ -262,7 +260,7 @@ class CommandTable(Table): """ def __getitem__(self, item): - if isinstance(item, six.string_types): + if isinstance(item, str): if item in self.colnames: return self.columns[item] else: @@ -294,7 +292,7 @@ def __getitem__(self, item): raise ValueError('Illegal type {0} for table item access' .format(type(item))) - def __unicode__(self): + def __str__(self): # Cut out params column for printing colnames = self.colnames if 'idx' in colnames: @@ -320,13 +318,8 @@ def __unicode__(self): lines = tmp.pformat(max_width=-1) return '\n'.join(lines) - if not six.PY2: - __str__ = __unicode__ - def __bytes__(self): - return six.text_type(self).encode('utf-8') - if six.PY2: - __str__ = __bytes__ + return str(self).encode('utf-8') def fetch_params(self): """ diff --git a/kadi/commands/states.py b/kadi/commands/states.py index 094ec01a..b37f4f6c 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -9,8 +9,6 @@ import inspect import numpy as np -import six -from six.moves import range from astropy.table import Table, Column @@ -114,11 +112,11 @@ def __new__(mcls, name, bases, members): return cls -@six.add_metaclass(TransitionMeta) class BaseTransition(object): """ Base transition class from which all actual transition classes are derived. """ + __metaclass__ = TransitionMeta @classmethod def get_state_changing_commands(cls, cmds): @@ -992,7 +990,7 @@ def get_transition_classes(state_keys=None): Get all BaseTransition subclasses in this module corresponding to state keys ``state_keys``. """ - if isinstance(state_keys, six.string_types): + if isinstance(state_keys, str): state_keys = [state_keys] elif state_keys is None: state_keys = DEFAULT_STATE_KEYS diff --git a/kadi/events/json_field.py b/kadi/events/json_field.py index e7bd9b84..926e748b 100644 --- a/kadi/events/json_field.py +++ b/kadi/events/json_field.py @@ -21,8 +21,6 @@ class LOL(models.Model): from django.conf import settings from django.utils import simplejson -import six - class JSONEncoder(simplejson.JSONEncoder): def default(self, obj): @@ -66,10 +64,10 @@ def __repr__(self): return dumps(self) -@six.add_metaclass(models.SubfieldBase) class JSONField(models.TextField): """JSONField is a generic textfield that neatly serializes/unserializes JSON objects seamlessly. Main thingy must be a dict object.""" + __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): default = kwargs.get('default') @@ -83,7 +81,7 @@ def to_python(self, value): """Convert our string value to JSON after we load it from the DB""" if value is None or value == '': return {} - elif isinstance(value, six.string_types): + elif isinstance(value, str): res = loads(value) if isinstance(res, dict): return JSONDict(**res) diff --git a/kadi/events/models.py b/kadi/events/models.py index 779f832f..aa86d704 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -3,16 +3,13 @@ import operator from itertools import count -from six.moves import zip -import six from django.db import models -models.query.REPR_OUTPUT_SIZE = 1000 # Increase default number of rows printed - import pyyaks.logger - from .manvr_templates import get_manvr_templates +models.query.REPR_OUTPUT_SIZE = 1000 # Increase default number of rows printed + # Fool pyflakes into thinking these are defined fetch = None interpolate = None @@ -285,7 +282,7 @@ def __repr__(self): @property def table(self): def un_unicode(vals): - return tuple(val.encode('ascii') if isinstance(val, six.text_type) else val + return tuple(val.encode('ascii') if isinstance(val, str) else val for val in vals) from astropy.table import Table @@ -293,8 +290,6 @@ def un_unicode(vals): model_fields = self.model.get_model_fields() names = [f.name for f in model_fields] rows = self.values_list() - if six.PY2: - rows = [un_unicode(vals) for vals in rows] cols = (list(zip(*rows)) if len(rows) > 0 else None) dat = Table(cols, names=names) @@ -518,14 +513,10 @@ def get_obsid(self): return obsids[0].obsid def __bytes__(self): - return six.text_type(self).encode('utf-8') + return str(self).encode('utf-8') - if six.PY2: - def __str__(self): - return self.__bytes__() - else: - def __str__(self): - return self.__unicode__() + def __str__(self): + return self.__unicode__() class BaseEvent(BaseModel): diff --git a/kadi/events/query.py b/kadi/events/query.py index c46c26a0..3c6583ef 100644 --- a/kadi/events/query.py +++ b/kadi/events/query.py @@ -3,8 +3,6 @@ import warnings import numpy as np -from six.moves import zip -import six from Chandra.Time import DateTime @@ -18,11 +16,7 @@ def un_unicode(vals): # I think this code is orphaned and should never be called. warnings.warn('unexpected call to query.un_unicode', stacklevel=2) - if six.PY2: - out = tuple(val.encode('ascii') if isinstance(val, six.text_type) else val - for val in vals) - else: - out = tuple(vals) + out = tuple(vals) return out diff --git a/kadi/events/views.py b/kadi/events/views.py index 20676096..ad75d5c4 100644 --- a/kadi/events/views.py +++ b/kadi/events/views.py @@ -1,5 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from six.moves import urllib, range +import urllib import shlex import re diff --git a/kadi/occweb.py b/kadi/occweb.py index 45329e6d..451d43ff 100644 --- a/kadi/occweb.py +++ b/kadi/occweb.py @@ -11,7 +11,6 @@ import time from collections import OrderedDict as odict -import six import numpy as np import requests @@ -92,7 +91,7 @@ def get_ifot(event_type, start=None, stop=None, props=[], columns=[], timeout=TI response = requests.get(url, auth=get_auth(), params=params, timeout=timeout) # For Py2 convert from unicode to ASCII str - text = response.text.encode('ascii', 'ignore') if six.PY2 else response.text + text = response.text text = re.sub(r'\r\n', ' ', text) lines = [x for x in text.split('\t\n') if x.strip()] diff --git a/kadi/settings.py b/kadi/settings.py index cfce3762..977366fa 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -38,7 +38,7 @@ BASE_DIR = dirname(dirname(realpath(__file__))) # Data paths for kadi project -from .paths import EVENTS_DB_PATH, DATA_DIR +from .paths import EVENTS_DB_PATH, DATA_DIR # noqa # Make sure there is an events database if not os.path.exists(EVENTS_DB_PATH()): diff --git a/kadi/update_events.py b/kadi/update_events.py index d771effe..2bf0c475 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -7,7 +7,6 @@ import time import numpy as np -import six from . import occweb import pyyaks.logger @@ -260,7 +259,7 @@ def main(): # Get the event classes in models module EventModels = [Model for name, Model in vars(models).items() - if (isinstance(Model, six.class_types) # is a class + if (isinstance(Model, type) # is a class and issubclass(Model, models.BaseEvent) # is a BaseEvent subclass and 'Meta' not in Model.__dict__ # is not a base class and hasattr(Model, 'get_events') # can get events From b26e7fbab6b517a0830fd651a1ae0fe83e4991c8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Dec 2019 20:46:09 -0500 Subject: [PATCH 18/35] Fix test failure --- kadi/tests/test_events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadi/tests/test_events.py b/kadi/tests/test_events.py index 2d32a58d..35fd069b 100644 --- a/kadi/tests/test_events.py +++ b/kadi/tests/test_events.py @@ -153,7 +153,8 @@ def test_intervals_filter(): # 2000-05-01 00:00:00 | 3SDTSTSV | Y # 2000-05-13 00:00:00 | 3SDP15V | 1 - assert (str(ltt_bads().filter('2000:121', '2000:134')).splitlines() == + lines = sorted(str(ltt_bads().filter('2000:121', '2000:134')).splitlines()) + assert (lines == ['', '', '', From 0d447c3a1b9df72c254788f38f84315cb506c41e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Dec 2019 21:13:52 -0500 Subject: [PATCH 19/35] Fix mistake in six-removal re metaclasses --- kadi/commands/states.py | 4 +--- kadi/events/json_field.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/kadi/commands/states.py b/kadi/commands/states.py index b37f4f6c..cd9ce0da 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -112,12 +112,10 @@ def __new__(mcls, name, bases, members): return cls -class BaseTransition(object): +class BaseTransition(object, metaclass=TransitionMeta): """ Base transition class from which all actual transition classes are derived. """ - __metaclass__ = TransitionMeta - @classmethod def get_state_changing_commands(cls, cmds): """ diff --git a/kadi/events/json_field.py b/kadi/events/json_field.py index 926e748b..5f0acbb9 100644 --- a/kadi/events/json_field.py +++ b/kadi/events/json_field.py @@ -64,10 +64,9 @@ def __repr__(self): return dumps(self) -class JSONField(models.TextField): +class JSONField(models.TextField, metaclass=models.SubfieldBase): """JSONField is a generic textfield that neatly serializes/unserializes JSON objects seamlessly. Main thingy must be a dict object.""" - __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): default = kwargs.get('default') From c4a98d81e9294792bff3236e7be304516745a229 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Dec 2019 20:43:11 -0500 Subject: [PATCH 20/35] Fix an issue with non-UTF8 char in TLR files --- kadi/events/orbit_funcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/events/orbit_funcs.py b/kadi/events/orbit_funcs.py index d1cd0d61..763f9b49 100644 --- a/kadi/events/orbit_funcs.py +++ b/kadi/events/orbit_funcs.py @@ -177,7 +177,7 @@ def get_orbit_points(tlrfiles): # 012345678901234567890123456789012345678901234567890123456789 logger.info('Getting points from {}'.format(tlrfile)) try: - fh = open(tlrfile, 'r') + fh = open(tlrfile, 'r', encoding='ascii', errors='ignore') except IOError as err: logger.warn(err) continue From 9fc3c79f9ffdebfca526b9cb39e92240ddfa495b Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 7 Dec 2019 16:10:24 -0500 Subject: [PATCH 21/35] Remove useless comment --- kadi/events/urls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kadi/events/urls.py b/kadi/events/urls.py index 63b1da59..123b2ca8 100644 --- a/kadi/events/urls.py +++ b/kadi/events/urls.py @@ -31,6 +31,5 @@ except KeyError: pass -urlpatterns = [# '', - url('^$', views.IndexView.as_view()), +urlpatterns = [url('^$', views.IndexView.as_view()), *urls] From ca552b7f481969c8f496e29fb1a17c459525f705 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 7 Dec 2019 17:30:04 -0500 Subject: [PATCH 22/35] Change events database name to events3.db3 --- kadi/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/paths.py b/kadi/paths.py index 16080ed3..0bb7f3e2 100644 --- a/kadi/paths.py +++ b/kadi/paths.py @@ -11,7 +11,7 @@ def DATA_DIR(): def EVENTS_DB_PATH(): - return os.path.join(DATA_DIR(), 'events.db3') + return os.path.join(DATA_DIR(), 'events3.db3') def IDX_CMDS_PATH(): From 1326587eb99b89978a5300117023bb8809654af9 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 7 Dec 2019 17:30:38 -0500 Subject: [PATCH 23/35] Add WARNING in case of obsid not available --- kadi/events/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index aa86d704..95fa8d8a 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -413,7 +413,8 @@ def from_dict(cls, model_dict, logger=None): tstart = DateTime(model_dict[cls._get_obsid_start_attr]).secs obsrq = fetch.Msid('cobsrqid', tstart, tstart + 200) if len(obsrq.vals) == 0: - logger.warn(f'unable to get COBSRQID near {model_dict[cls._get_obsid_start_attr]}, ' + logger.warn(f'WARNING: unable to get COBSRQID near ' + f'{model_dict[cls._get_obsid_start_attr]}, ' f'using obsid=-999') model_dict['obsid'] = -999 else: From 9e4d108c6189b0b3984ac48d8a5166a5cc5bf860 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 8 Dec 2019 06:52:08 -0500 Subject: [PATCH 24/35] Move ltt_bads into share and other setup changes --- kadi/events/models.py | 11 +++++++---- setup.py | 6 ++++-- task_schedule_events.cfg | 6 +++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/kadi/events/models.py b/kadi/events/models.py index 95fa8d8a..caaf60e3 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -1,6 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -import os +import sys import operator +from pathlib import Path from itertools import count @@ -2330,8 +2331,10 @@ def get_events(cls, start, stop=None): start = DateTime(start).date stop = DateTime(stop).date - filename = os.path.join(DATA_DIR(), cls.intervals_file) - intervals = table.Table.read(filename, **cls.table_read_kwargs) # noqa + filename = Path(cls.intervals_file) + if not filename.absolute(): + filename = Path(DATA_DIR(), filename) + intervals = table.Table.read(str(filename), **cls.table_read_kwargs) # noqa # Custom in-place processing of raw intervals cls.process_intervals(intervals) @@ -2394,7 +2397,7 @@ class LttBad(AsciiTableEvent): key._kadi_hidden = True dur._kadi_format = '{:.1f}' - intervals_file = 'ltt_bads.dat' + intervals_file = Path(sys.prefix) / 'share' / 'kadi' / 'ltt_bads.dat' # Table.read keyword args table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', guess=False) start_column = 'start' diff --git a/setup.py b/setup.py index 786a7fff..4ca844c2 100644 --- a/setup.py +++ b/setup.py @@ -21,13 +21,15 @@ if "--user" not in sys.argv: share_path = os.path.join(sys.prefix, "share", "kadi") data_files = [(share_path, ['task_schedule_cmds.cfg', - 'task_schedule_events.cfg'])] + 'task_schedule_events.cfg', + 'ltt_bads.dat'])] else: data_files = None entry_points = {'console_scripts': [ 'get_chandra_states = kadi.commands.states:get_chandra_states', - 'kadi_update_cmds = kadi.update_cmds:main']} + 'kadi_update_cmds = kadi.update_cmds:main', + 'kadi_update_events = kadi.update_events:main']} setup(name='kadi', use_scm_version=True, diff --git a/task_schedule_events.cfg b/task_schedule_events.cfg index 97651e05..4e2c90eb 100644 --- a/task_schedule_events.cfg +++ b/task_schedule_events.cfg @@ -42,9 +42,9 @@ alert aca@head.cfa.harvard.edu cron * * * * * check_cron * * * * * exec /bin/mkdir -p $ENV{SKA_DATA}/kadi/update_events - exec /bin/cp $ENV{SKA_DATA}/kadi/events.db3 $ENV{SKA_DATA}/kadi/ltt_bads.dat $ENV{SKA_DATA}/kadi/update_events/ - exec update_events --data-root=$ENV{SKA_DATA}/kadi/update_events - exec /bin/mv $ENV{SKA_DATA}/kadi/update_events/events.db3 $ENV{SKA_DATA}/kadi/events.db3 + exec /bin/cp $ENV{SKA_DATA}/kadi/events3.db3 $ENV{SKA_DATA}/kadi/update_events/ + exec kadi_update_events --data-root=$ENV{SKA_DATA}/kadi/update_events + exec /bin/mv $ENV{SKA_DATA}/kadi/update_events/events3.db3 $ENV{SKA_DATA}/kadi/events3.db3 # File Expression From 4daddb897711265671116fbea1954d14df15730f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 19 Dec 2019 15:39:16 -0500 Subject: [PATCH 25/35] Fix to work with find_attitude --- kadi/settings.py | 13 ++----------- kadi/urls.py | 4 ++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/kadi/settings.py b/kadi/settings.py index 977366fa..b077def7 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -96,25 +96,16 @@ INSTALLED_APPS = ( 'django.contrib.admin', - # 'django.contrib.admindocs', # ?? needed? Lost in 1.6 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'kadi.events', + 'mica.web', + 'find_attitude.web_find_attitude', # app label (last module) must be unique ) -OPTIONAL_APPS = ('mica.web', 'find_attitude.web') -for app in OPTIONAL_APPS: - try: - __import__(app) - except ImportError: - pass - else: - INSTALLED_APPS += (app,) - - MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', diff --git a/kadi/urls.py b/kadi/urls.py index f0b78b9f..933e56cc 100644 --- a/kadi/urls.py +++ b/kadi/urls.py @@ -4,7 +4,7 @@ import kadi.views import mica.web.views -# import find_attitude.web.views +import find_attitude.web_find_attitude.views admin.autodiscover() @@ -15,5 +15,5 @@ url(r'^mica/$', mica.web.views.IndexView.as_view()), url(r'^pcad_acq/$', mica.web.views.AcqView.as_view()), url(r'^star_hist/$', mica.web.views.StarHistView.as_view()), - # url(r'^find_attitude/$', find_attitude.web.views.index), + url(r'^find_attitude/$', find_attitude.web_find_attitude.views.index), ] From f68ada509a4606be2ea65738fd5911986b324ee5 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 19 Dec 2019 16:21:53 -0500 Subject: [PATCH 26/35] Fixes for standalone / macosx / zsh --- kadi/occweb.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/kadi/occweb.py b/kadi/occweb.py index 451d43ff..e6810a20 100644 --- a/kadi/occweb.py +++ b/kadi/occweb.py @@ -10,6 +10,7 @@ import hashlib import time from collections import OrderedDict as odict +from pathlib import Path import numpy as np import requests @@ -29,12 +30,16 @@ def get_auth(): - authfile = '/proj/sot/ska/data/aspect_authorization/occweb-{}'.format(os.environ['USER']) - config = configobj.ConfigObj(authfile) - username = config.get('username') - password = config.get('password') - - # If /proj/sot/ska doesn't have occweb credentials try .netrc. + username = None + ska = os.environ.get('SKA') + if ska: + user = os.environ.get('USER') or os.environ.get('LOGNAME') + authfile = Path(ska, 'data', 'aspect_authorization', f'occweb-{user}') + config = configobj.ConfigObj(str(authfile)) + username = config.get('username') + password = config.get('password') + + # If $SKA doesn't have occweb credentials try .netrc. if username is None: try: import Ska.ftp From 1d48cceea194ab1dfd552f2df790dc2827f757e6 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 28 Dec 2019 05:54:10 -0500 Subject: [PATCH 27/35] More changes for modern env and django 3.0 --- NOTES.build | 7 +++++-- kadi/events/models.py | 3 ++- kadi/settings.py | 7 +++++-- kadi/update_events.py | 19 ------------------- setup.py | 2 +- 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/NOTES.build b/NOTES.build index 812ccd0a..6d16b76f 100644 --- a/NOTES.build +++ b/NOTES.build @@ -4,11 +4,14 @@ cd ~/git/kadi export KADI=$PWD -rm events.db3 cmds.h5 cmds.pkl +rm -f events3.db3 cmds.h5 cmds.pkl +rm -rf kadi/events/migrations ./manage.py makemigrations events ./manage.py migrate -python -m kadi.update_events --start=1999:200 --stop=2000:001 --model=SafeSun +# First line is just to see that every model works. One can just drop the +# --stop=2000:001 if you are sure it will work. +python -m kadi.update_events --start=1999:240 --stop=2000:001 python -m kadi.update_events --start=2000:001 python -m kadi.update_cmds --start=2000:001 diff --git a/kadi/events/models.py b/kadi/events/models.py index caaf60e3..569ddebf 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -2399,7 +2399,8 @@ class LttBad(AsciiTableEvent): intervals_file = Path(sys.prefix) / 'share' / 'kadi' / 'ltt_bads.dat' # Table.read keyword args - table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', guess=False) + table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', + guess=False, fill_values=()) start_column = 'start' stop_column = 'stop' diff --git a/kadi/settings.py b/kadi/settings.py index b077def7..03e888cc 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -87,8 +87,6 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -TEMPLATE_DEBUG = True - ALLOWED_HOSTS = [] @@ -181,6 +179,7 @@ ], 'APP_DIRS': True, 'OPTIONS': { + 'debug': True, 'context_processors': [ # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this # list if you haven't customized them: @@ -195,3 +194,7 @@ }, }, ] +# (1_8.W001) The standalone TEMPLATE_* settings were deprecated in Django 1.8 +# and the TEMPLATES dictionary takes precedence. You must put the values of the +# following settings into your default TEMPLATES dict: TEMPLATE_DEBUG. + diff --git a/kadi/update_events.py b/kadi/update_events.py index 2bf0c475..e233c14a 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -8,7 +8,6 @@ import numpy as np -from . import occweb import pyyaks.logger from Chandra.Time import DateTime @@ -16,7 +15,6 @@ def get_opt(args=None): - OCC_SOT_ACCOUNT = os.environ['USER'].lower() == 'sot' parser = argparse.ArgumentParser(description='Update the events database') parser.add_argument("--stop", default=DateTime().date, @@ -43,14 +41,6 @@ def get_opt(args=None): parser.add_argument("--data-root", default=".", help="Root data directory (default='.')") - parser.add_argument("--occ", - default=OCC_SOT_ACCOUNT, - action='store_true', - help="Running at OCC as copy-only client") - parser.add_argument("--ftp", - default=False, - action='store_true', - help="Store or get files via ftp (implied for --occ)") args = parser.parse_args(args) return args @@ -238,11 +228,6 @@ def main(): for line in pformat(vars(opt)).splitlines(): logger.info(' {}'.format(line)) - if opt.occ: - # Get events database file from HEAD via lucky ftp - occweb.ftp_get_from_lucky('kadi', [EVENTS_DB_PATH()], logger=logger) - return - from .events import models # Allow for a cmd line option --start. If supplied then loop the @@ -286,10 +271,6 @@ def main(): logger.error(f'ERROR in processing {EventModel}') logger.error(f'Traceback:\n{traceback.format_exc()}') - if opt.ftp: - # Push events database file to OCC via lucky ftp - occweb.ftp_put_to_lucky('kadi', [EVENTS_DB_PATH()], logger=logger) - if __name__ == '__main__': main() diff --git a/setup.py b/setup.py index 4ca844c2..97339d7a 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ cmdclass = {} if "--user" not in sys.argv: - share_path = os.path.join(sys.prefix, "share", "kadi") + share_path = os.path.join("share", "kadi") data_files = [(share_path, ['task_schedule_cmds.cfg', 'task_schedule_events.cfg', 'ltt_bads.dat'])] From a551bacfe920a0fae440295c399b297f4afc8435 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 28 Dec 2019 06:09:25 -0500 Subject: [PATCH 28/35] Autopep8 W504 move binary operator after line break --- kadi/cmds/cmds.py | 8 ++++---- kadi/commands/commands.py | 12 ++++++------ kadi/commands/states.py | 12 ++++++------ kadi/commands/tests/test_states.py | 11 ++++++----- kadi/events/models.py | 4 ++-- kadi/events/query.py | 4 ++-- kadi/settings.py | 1 - kadi/tests/test_events.py | 16 ++++++++-------- kadi/update_cmds.py | 8 ++++---- 9 files changed, 38 insertions(+), 38 deletions(-) diff --git a/kadi/cmds/cmds.py b/kadi/cmds/cmds.py index 62d12954..a0bb5558 100644 --- a/kadi/cmds/cmds.py +++ b/kadi/cmds/cmds.py @@ -171,16 +171,16 @@ def __init__(self, cmd): if self['tlmsid'] == 'None': colnames.remove('tlmsid') - self._ordered_keys = (colnames[1:] + - [par[0] for par in rev_pars_dict[cmd['idx']]]) + self._ordered_keys = (colnames[1:] + + [par[0] for par in rev_pars_dict[cmd['idx']]]) def __repr__(self): out = ('<{} '.format(self.__class__.__name__) + str(self) + '>') return out def __str__(self): - out = ('{} {:11s} '.format(self['date'], self['type']) + - ' '.join('{}={}'.format(key, self[key]) for key in self._ordered_keys + out = ('{} {:11s} '.format(self['date'], self['type']) + + ' '.join('{}={}'.format(key, self[key]) for key in self._ordered_keys if key not in ('type', 'date'))) return out diff --git a/kadi/commands/commands.py b/kadi/commands/commands.py index 25a1be89..aeed3aef 100644 --- a/kadi/commands/commands.py +++ b/kadi/commands/commands.py @@ -244,8 +244,8 @@ def __str__(self): if 'idx' in keys: keys.remove('idx') - out = ('{} {} '.format(self['date'], self['type']) + - ' '.join('{}={}'.format(key, self[key]) for key in keys + out = ('{} {} '.format(self['date'], self['type']) + + ' '.join('{}={}'.format(key, self[key]) for key in keys if key not in ('type', 'date'))) return out @@ -278,10 +278,10 @@ def __getitem__(self, item): keys=self.groups._keys) return out - elif (isinstance(item, slice) or - isinstance(item, np.ndarray) or - isinstance(item, list) or - isinstance(item, tuple) and all(isinstance(x, np.ndarray) + elif (isinstance(item, slice) + or isinstance(item, np.ndarray) + or isinstance(item, list) + or isinstance(item, tuple) and all(isinstance(x, np.ndarray) for x in item)): # here for the many ways to give a slice; a tuple of ndarray # is produced by np.where, as in t[np.where(t['a'] > 2)] diff --git a/kadi/commands/states.py b/kadi/commands/states.py index cd9ce0da..688a83aa 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -38,10 +38,10 @@ # State keys for PCAD-related transitions. If *any* of these are requested then # *all* of them need to be processed to get the correct answer. -PCAD_STATE_KEYS = (QUAT_COMPS + - ['targ_' + qc for qc in QUAT_COMPS] + - ['ra', 'dec', 'roll'] + - ['auto_npnt', 'pcad_mode', 'pitch', 'off_nom_roll']) +PCAD_STATE_KEYS = (QUAT_COMPS + + ['targ_' + qc for qc in QUAT_COMPS] + + ['ra', 'dec', 'roll'] + + ['auto_npnt', 'pcad_mode', 'pitch', 'off_nom_roll']) # Default state keys (mostly matches classic command states list) DEFAULT_STATE_KEYS = ('ccd_count', 'clocking', 'dec', 'dither', 'fep_count', @@ -188,8 +188,8 @@ def _auto_update_docstring(cls): others = [] for attr, val in cls.__dict__.items(): - if (attr.startswith('_') or inspect.ismethod(val) or - attr in ('state_keys', 'command_attributes', 'command_params')): + if (attr.startswith('_') or inspect.ismethod(val) + or attr in ('state_keys', 'command_attributes', 'command_params')): continue others.append('{}={}'.format(attr, val)) if others: diff --git a/kadi/commands/tests/test_states.py b/kadi/commands/tests/test_states.py index 4c858d87..989411ff 100644 --- a/kadi/commands/tests/test_states.py +++ b/kadi/commands/tests/test_states.py @@ -108,9 +108,9 @@ def test_quick(): """ state_keys = (['obsid', 'clocking', 'power_cmd', 'fep_count', 'vid_board', 'si_mode', 'ccd_count'] + - ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] + - ['letg', 'hetg'] + - ['simpos', 'simfa_pos']) + ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] + + ['letg', 'hetg'] + + ['simpos', 'simfa_pos']) continuity = {'letg': 'RETR', 'hetg': 'RETR'} # Not necessarily set within 7 days rc, rk = compare_states('2018:235', '2018:245', state_keys, continuity=continuity) @@ -154,6 +154,7 @@ def test_states_2017(): maneuver commanding (so the NMAN period before maneuver starts is OK) while kadi will insert pitch breaks only in NPNT. (Tested later). """ + state_keys = (['obsid', 'clocking', 'power_cmd', 'fep_count'] + ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] + ['letg', 'hetg'] @@ -442,8 +443,8 @@ def test_reduce_states_cmd_states(): cs = cmd_states.fetch_states('2018:235', '2018:245', allow_identical=True) cs = Table(cs) - state_keys = (set(cmd_states.STATE0) - - set(['datestart', 'datestop', 'trans_keys', 'tstart', 'tstop'])) + state_keys = (set(cmd_states.STATE0) + - set(['datestart', 'datestop', 'trans_keys', 'tstart', 'tstop'])) # Default setting is reduce states with merge_identical=False, which is the same # as cmd_states. diff --git a/kadi/events/models.py b/kadi/events/models.py index 569ddebf..2eeae602 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -1550,8 +1550,8 @@ def get_events(cls, start, stop=None): i1 = np.searchsorted(changes['time'], manvr_next['tstart']) sequence = changes[i0:i1 + 1] sequence['dt'] = (sequence['time'] + sequence['prev_time']) / 2.0 - manvr['tstop'] - ok = ((sequence['dt'] >= ZERO_DT) | (sequence['msid'] == 'aofattmd') | - (sequence['msid'] == 'aopcadmd')) + ok = ((sequence['dt'] >= ZERO_DT) | (sequence['msid'] == 'aofattmd') + | (sequence['msid'] == 'aopcadmd')) sequence = sequence[ok] manvr_attrs = cls.get_manvr_attrs(sequence) diff --git a/kadi/events/query.py b/kadi/events/query.py index 3c6583ef..5ae9f7d8 100644 --- a/kadi/events/query.py +++ b/kadi/events/query.py @@ -169,8 +169,8 @@ def __repr__(self): if self.interval_pad.start != 0: bits.append(' pad={}'.format(self.interval_pad.start)) if self.filter_kwargs: - bits.append(' ' + - ' '.join('{}={!r}'.format(k, v) for k, v in self.filter_kwargs.items())) + bits.append(' ' + + ' '.join('{}={!r}'.format(k, v) for k, v in self.filter_kwargs.items())) bits.append('>') return ''.join(bits) diff --git a/kadi/settings.py b/kadi/settings.py index 03e888cc..f3ec56af 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -197,4 +197,3 @@ # (1_8.W001) The standalone TEMPLATE_* settings were deprecated in Django 1.8 # and the TEMPLATES dictionary takes precedence. You must put the values of the # following settings into your default TEMPLATES dict: TEMPLATE_DEBUG. - diff --git a/kadi/tests/test_events.py b/kadi/tests/test_events.py index 35fd069b..79374278 100644 --- a/kadi/tests/test_events.py +++ b/kadi/tests/test_events.py @@ -154,22 +154,22 @@ def test_intervals_filter(): # 2000-05-13 00:00:00 | 3SDP15V | 1 lines = sorted(str(ltt_bads().filter('2000:121', '2000:134')).splitlines()) - assert (lines == - ['', + assert (lines + == ['', '', '', '']) # No filter - assert (ltt_bads.intervals(start, stop) == - [('2000:121:12:00:00.000', '2000:123:00:00:00.000'), + assert (ltt_bads.intervals(start, stop) + == [('2000:121:12:00:00.000', '2000:123:00:00:00.000'), ('2000:134:00:00:00.000', '2000:134:12:00:00.000')]) - assert (ltt_bads(flag='Y').intervals(start, stop) == - [('2000:122:00:00:00.000', '2000:123:00:00:00.000')]) + assert (ltt_bads(flag='Y').intervals(start, stop) + == [('2000:122:00:00:00.000', '2000:123:00:00:00.000')]) - assert (ltt_bads(msid='ELBI_LOW', flag='R').intervals(start, stop) == - [('2000:121:12:00:00.000', '2000:122:00:00:00.000')]) + assert (ltt_bads(msid='ELBI_LOW', flag='R').intervals(start, stop) + == [('2000:121:12:00:00.000', '2000:122:00:00:00.000')]) assert (ltt_bads(msid='ELBI_LOW', flag='1').intervals(start, stop) == []) diff --git a/kadi/update_cmds.py b/kadi/update_cmds.py index 452a337f..bba1b182 100644 --- a/kadi/update_cmds.py +++ b/kadi/update_cmds.py @@ -189,8 +189,8 @@ def get_cmds(start, stop, mp_dir='/data/mpcrit1/mplogs'): # Only store commands for this timeline (match SCS and date) bs_cmds = [x for x in bs_cmds - if tl['datestart'] <= x['date'] <= tl['datestop'] and - x['scs'] == tl['scs']] + if tl['datestart'] <= x['date'] <= tl['datestop'] + and x['scs'] == tl['scs']] for bs_cmd in bs_cmds: bs_cmd['timeline_id'] = tl['id'] @@ -232,8 +232,8 @@ def get_unique_orbit_cmds(orbit_cmds): # "different" from uniq_cmds[-1]. for cmd in orbit_cmds: last_cmd = uniq_cmds[-1] - if (cmd['params']['EVENT_TYPE'] == last_cmd['params']['EVENT_TYPE'] and - abs(DateTime(cmd['date']).secs - DateTime(last_cmd['date']).secs) < 180): + if (cmd['params']['EVENT_TYPE'] == last_cmd['params']['EVENT_TYPE'] + and abs(DateTime(cmd['date']).secs - DateTime(last_cmd['date']).secs) < 180): # Same event as last (even if date is a bit different). Now if this one # has a larger timeline_id that means it is from a more recent schedule, so # use that one. From 07427a6d0ed4384aaa559b8d1d1ce8579aeec316 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 28 Dec 2019 06:16:01 -0500 Subject: [PATCH 29/35] More flake8 changes --- kadi/cmds/cmds.py | 2 +- kadi/commands/commands.py | 4 ++-- kadi/commands/tests/test_states.py | 4 ++-- kadi/events/models.py | 8 ++++---- kadi/events/query.py | 5 +++-- kadi/tests/test_events.py | 8 ++++---- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/kadi/cmds/cmds.py b/kadi/cmds/cmds.py index a0bb5558..577017cd 100644 --- a/kadi/cmds/cmds.py +++ b/kadi/cmds/cmds.py @@ -181,7 +181,7 @@ def __repr__(self): def __str__(self): out = ('{} {:11s} '.format(self['date'], self['type']) + ' '.join('{}={}'.format(key, self[key]) for key in self._ordered_keys - if key not in ('type', 'date'))) + if key not in ('type', 'date'))) return out diff --git a/kadi/commands/commands.py b/kadi/commands/commands.py index aeed3aef..e6769d8e 100644 --- a/kadi/commands/commands.py +++ b/kadi/commands/commands.py @@ -246,7 +246,7 @@ def __str__(self): out = ('{} {} '.format(self['date'], self['type']) + ' '.join('{}={}'.format(key, self[key]) for key in keys - if key not in ('type', 'date'))) + if key not in ('type', 'date'))) return out def __sstr__(self): @@ -282,7 +282,7 @@ def __getitem__(self, item): or isinstance(item, np.ndarray) or isinstance(item, list) or isinstance(item, tuple) and all(isinstance(x, np.ndarray) - for x in item)): + for x in item)): # here for the many ways to give a slice; a tuple of ndarray # is produced by np.where, as in t[np.where(t['a'] > 2)] # For all, a new table is constructed with slice of all columns diff --git a/kadi/commands/tests/test_states.py b/kadi/commands/tests/test_states.py index 989411ff..359fc0c8 100644 --- a/kadi/commands/tests/test_states.py +++ b/kadi/commands/tests/test_states.py @@ -107,8 +107,8 @@ def test_quick(): Test for a few days in 2017. Sanity check for refactoring etc. """ state_keys = (['obsid', 'clocking', 'power_cmd', 'fep_count', 'vid_board', - 'si_mode', 'ccd_count'] + - ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] + 'si_mode', 'ccd_count'] + + ['q1', 'q2', 'q3', 'q4', 'pcad_mode', 'dither', 'ra', 'dec', 'roll'] + ['letg', 'hetg'] + ['simpos', 'simfa_pos']) continuity = {'letg': 'RETR', 'hetg': 'RETR'} # Not necessarily set within 7 days diff --git a/kadi/events/models.py b/kadi/events/models.py index 2eeae602..cde658bf 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -1354,8 +1354,8 @@ def get_dwells(cls, event, changes): # End of dwell because of NPNT => NMAN transition OR another acquisition elif (state == 'dwell' - and ((change['msid'] == 'aopcadmd' and change['val'] == 'NMAN') or - (change['msid'] == 'aoacaseq' and change['time'] - t0 > 400))): + and ((change['msid'] == 'aopcadmd' and change['val'] == 'NMAN') + or (change['msid'] == 'aoacaseq' and change['time'] - t0 > 400))): dwell['tstop'] = change['prev_time'] dwell['stop'] = change['prev_date'] dwell['dur'] = dwell['tstop'] - dwell['tstart'] @@ -1415,8 +1415,8 @@ def match(msid, val, idx=None, filter=None): # least twice as of ~Mar 2012. manvr_templates = get_manvr_templates() seqs = ['{}_{}_{}'.format(c['msid'], c['prev_val'], c['val']) for c in changes - if (c['msid'] in ('aopcadmd', 'aofattmd', 'aoacaseq') and - c['dt'] >= ZERO_DT)] + if (c['msid'] in ('aopcadmd', 'aofattmd', 'aoacaseq') + and c['dt'] >= ZERO_DT)] for name, manvr_template in manvr_templates: if seqs == manvr_template[2:]: # skip first two which are STDY-MNVR and MNVR-STDY template = name diff --git a/kadi/events/query.py b/kadi/events/query.py index 5ae9f7d8..e4dfc6d6 100644 --- a/kadi/events/query.py +++ b/kadi/events/query.py @@ -169,8 +169,9 @@ def __repr__(self): if self.interval_pad.start != 0: bits.append(' pad={}'.format(self.interval_pad.start)) if self.filter_kwargs: - bits.append(' ' - + ' '.join('{}={!r}'.format(k, v) for k, v in self.filter_kwargs.items())) + bits.append( + ' ' + ' '.join('{}={!r}'.format(k, v) + for k, v in self.filter_kwargs.items())) bits.append('>') return ''.join(bits) diff --git a/kadi/tests/test_events.py b/kadi/tests/test_events.py index 79374278..2b1c7e62 100644 --- a/kadi/tests/test_events.py +++ b/kadi/tests/test_events.py @@ -156,14 +156,14 @@ def test_intervals_filter(): lines = sorted(str(ltt_bads().filter('2000:121', '2000:134')).splitlines()) assert (lines == ['', - '', - '', - '']) + '', + '', + '']) # No filter assert (ltt_bads.intervals(start, stop) == [('2000:121:12:00:00.000', '2000:123:00:00:00.000'), - ('2000:134:00:00:00.000', '2000:134:12:00:00.000')]) + ('2000:134:00:00:00.000', '2000:134:12:00:00.000')]) assert (ltt_bads(flag='Y').intervals(start, stop) == [('2000:122:00:00:00.000', '2000:123:00:00:00.000')]) From 532988cc508244b2838a830727d20e5f794817b9 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 29 Dec 2019 08:19:36 -0500 Subject: [PATCH 30/35] Add logging information --- kadi/update_cmds.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kadi/update_cmds.py b/kadi/update_cmds.py index bba1b182..56d5e1a0 100644 --- a/kadi/update_cmds.py +++ b/kadi/update_cmds.py @@ -371,6 +371,8 @@ def main(args=None): logger = pyyaks.logger.get_logger(name='kadi', level=opt.log_level, format="%(asctime)s %(message)s") + logger.info('\n'.join(get_run_info(__file__, opt))) + # Set the global root data directory. This gets used in ..paths to # construct file names. The use of an env var is needed to allow # configurability of the root data directory within django. @@ -488,5 +490,21 @@ def read_backstop(filename): return bs +def get_run_info(filename, opt): + import time + import platform + from pprint import pformat + info = [f'******************************************', + f'Running {filename}', + f'Version: {__version__}', + f'Time: {time.ctime()}', + f'User: {os.getlogin()}', + f'Machine: {platform.node()}', + f'Processing args:', + pformat(vars(opt)), + f'******************************************'] + return info + + if __name__ == '__main__': main() From 3365d36f46c94958f85d1e44af15f1690912fb5e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 29 Dec 2019 08:19:57 -0500 Subject: [PATCH 31/35] Temporary change to alert to aldcroft not aca --- task_schedule_cmds.cfg | 2 +- task_schedule_events.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/task_schedule_cmds.cfg b/task_schedule_cmds.cfg index ebc36396..87efd1d2 100644 --- a/task_schedule_cmds.cfg +++ b/task_schedule_cmds.cfg @@ -20,7 +20,7 @@ heartbeat task_sched_heartbeat_cmds # running jobs (i.e. couldn't start jobs or couldn't open log file). # Processing errors *within* the jobs are caught with watch_cron_logs -alert aca@head.cfa.harvard.edu +alert aldcroft@head.cfa.harvard.edu # Define task parameters # cron: Job repetition specification ala crontab diff --git a/task_schedule_events.cfg b/task_schedule_events.cfg index 4e2c90eb..bdaa4151 100644 --- a/task_schedule_events.cfg +++ b/task_schedule_events.cfg @@ -21,7 +21,7 @@ heartbeat task_sched_heartbeat_events # running jobs (i.e. couldn't start jobs or couldn't open log file). # Processing errors *within* the jobs are caught with watch_cron_logs -alert aca@head.cfa.harvard.edu +alert aldcroft@head.cfa.harvard.edu # Define task parameters # cron: Job repetition specification ala crontab From 4ae68b74a849a632d4466f78d095302976beabd9 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 29 Dec 2019 08:31:35 -0500 Subject: [PATCH 32/35] Update build notes --- NOTES.build | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/NOTES.build b/NOTES.build index 6d16b76f..b9d050d6 100644 --- a/NOTES.build +++ b/NOTES.build @@ -9,11 +9,18 @@ rm -rf kadi/events/migrations ./manage.py makemigrations events ./manage.py migrate +# For commands one MUST do this in a dedicated test env because the pickling +# of UpdatedDict does not work. That object gets a module of __main__ but for +# production it must be kadi.update_cmds. See e.g. +# https://www.stefaanlippens.net/python-pickling-and-dealing-with-attributeerror- +# module-object-has-no-attribute-thing.html +pip install . # to a TEST env!! (Maybe with -e for editable install?) + # First line is just to see that every model works. One can just drop the # --stop=2000:001 if you are sure it will work. -python -m kadi.update_events --start=1999:240 --stop=2000:001 -python -m kadi.update_events --start=2000:001 -python -m kadi.update_cmds --start=2000:001 +kadi_update_events --start=1999:240 --stop=2000:001 +kadi_update_events --start=2000:001 +kadi_update_cmds --start=2000:001 ################################################################# # Re-build single table From 9909a828ebb224bc034eb2ffd9371eab4c1b7ad1 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 29 Dec 2019 14:38:27 -0500 Subject: [PATCH 33/35] Remove occ and ftp args; use ska_helpers.run_info --- kadi/update_cmds.py | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/kadi/update_cmds.py b/kadi/update_cmds.py index 56d5e1a0..7becda8b 100644 --- a/kadi/update_cmds.py +++ b/kadi/update_cmds.py @@ -13,7 +13,8 @@ import Ska.File from Chandra.Time import DateTime from Chandra.cmd_states.cmd_states import _tl_to_bs_cmds -from . import occweb +from ska_helpers.run_info import get_run_info_string + from .paths import IDX_CMDS_PATH, PARS_DICT_PATH from . import __version__ @@ -46,7 +47,6 @@ def get_opt(args=None): """ Get options for command line interface to update_cmd_states. """ - OCC_SOT_ACCOUNT = os.environ['USER'].lower() == 'sot' parser = argparse.ArgumentParser(description='Update HDF5 cmds table') parser.add_argument("--mp-dir", help=("MP load directory (default=/data/mpcrit1/mplogs) " @@ -62,14 +62,6 @@ def get_opt(args=None): parser.add_argument("--data-root", default='.', help="Data root (default='.')") - parser.add_argument("--occ", - default=OCC_SOT_ACCOUNT, - action='store_true', - help="Running at OCC as copy-only client") - parser.add_argument("--ftp", - default=False, - action='store_true', - help="Store or get files via ftp (implied for --occ)") parser.add_argument('--version', action='version', version='%(prog)s {version}'.format(version=__version__)) @@ -371,7 +363,7 @@ def main(args=None): logger = pyyaks.logger.get_logger(name='kadi', level=opt.log_level, format="%(asctime)s %(message)s") - logger.info('\n'.join(get_run_info(__file__, opt))) + logger.info('\n' + get_run_info_string(opt)) # Set the global root data directory. This gets used in ..paths to # construct file names. The use of an env var is needed to allow @@ -380,11 +372,6 @@ def main(args=None): idx_cmds_path = IDX_CMDS_PATH() pars_dict_path = PARS_DICT_PATH() - if opt.occ: - # Get cmds files from HEAD via lucky ftp - occweb.ftp_get_from_lucky('kadi', [idx_cmds_path, pars_dict_path], logger=logger) - return - try: with open(pars_dict_path, 'rb') as fh: pars_dict = pickle.load(fh) @@ -422,10 +409,6 @@ def main(args=None): else: logger.info('pars_dict was unmodified, not writing') - if opt.ftp: - # Push cmds files to OCC via lucky ftp - occweb.ftp_put_to_lucky('kadi', [idx_cmds_path, pars_dict_path], logger=logger) - def _coerce_type(val): """Coerce the supplied ``val`` (typically a string) into an int or float if @@ -490,21 +473,5 @@ def read_backstop(filename): return bs -def get_run_info(filename, opt): - import time - import platform - from pprint import pformat - info = [f'******************************************', - f'Running {filename}', - f'Version: {__version__}', - f'Time: {time.ctime()}', - f'User: {os.getlogin()}', - f'Machine: {platform.node()}', - f'Processing args:', - pformat(vars(opt)), - f'******************************************'] - return info - - if __name__ == '__main__': main() From 497efa174c561913a209f83fab50052f96fa0db8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 3 Jan 2020 12:05:54 -0500 Subject: [PATCH 34/35] Use ska_helpers log_run_info for run info in events update --- kadi/update_events.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kadi/update_events.py b/kadi/update_events.py index e233c14a..17fe1b32 100644 --- a/kadi/update_events.py +++ b/kadi/update_events.py @@ -10,6 +10,8 @@ import pyyaks.logger from Chandra.Time import DateTime +from ska_helpers.run_info import log_run_info +from kadi import __version__ # noqa logger = None # for pyflakes @@ -212,21 +214,15 @@ def main(): logger = pyyaks.logger.get_logger(name='kadi', level=opt.log_level, format="%(asctime)s %(message)s") + log_run_info(logger.info, opt) # Set the global root data directory. This gets used in the django # setup to find the sqlite3 database file. os.environ['KADI'] = os.path.abspath(opt.data_root) from .paths import EVENTS_DB_PATH - from . import __version__ - from pprint import pformat - logger.info('Kadi version : {}'.format(__version__)) - logger.info('Kadi path : {}'.format(os.path.dirname(os.path.abspath(__file__)))) logger.info('Event database : {}'.format(EVENTS_DB_PATH())) logger.info('') - logger.info('Options:') - for line in pformat(vars(opt)).splitlines(): - logger.info(' {}'.format(line)) from .events import models From 664c41bc3f64742c6e70282620634a362ca42874 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 3 Jan 2020 19:40:02 -0500 Subject: [PATCH 35/35] Fix leftover in use of run_info in update_cmds --- kadi/update_cmds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kadi/update_cmds.py b/kadi/update_cmds.py index 7becda8b..4ff9f4fe 100644 --- a/kadi/update_cmds.py +++ b/kadi/update_cmds.py @@ -13,7 +13,7 @@ import Ska.File from Chandra.Time import DateTime from Chandra.cmd_states.cmd_states import _tl_to_bs_cmds -from ska_helpers.run_info import get_run_info_string +from ska_helpers.run_info import log_run_info from .paths import IDX_CMDS_PATH, PARS_DICT_PATH from . import __version__ @@ -363,7 +363,7 @@ def main(args=None): logger = pyyaks.logger.get_logger(name='kadi', level=opt.log_level, format="%(asctime)s %(message)s") - logger.info('\n' + get_run_info_string(opt)) + log_run_info(logger.info, opt) # Set the global root data directory. This gets used in ..paths to # construct file names. The use of an env var is needed to allow