Handles Maneuver-only loads, improve regression testing

MANIFEST.in

@@ -1,2 +1,3 @@
 include acis_thermal_check/templates/*
-include acis_thermal_check/data/*
+include acis_thermal_check/data/*
+include acis_thermal_check/data/nlets/*

acis_thermal_check/conftest.py

@@ -5,7 +5,17 @@ def pytest_addoption(parser):
     parser.addoption("--answer_store", action='store_true',
                      help="If true, generate new answers, but don't test. "
                           "Default: False, which performs only the test.")
+    parser.addoption("--test_root", type=str,
+                     help="If specified, this will be the location which "
+                          "the test artifacts will be stored. If not, a "
+                          "temporary directory is created.")
+
 
 @pytest.fixture()
 def answer_store(request):
     return request.config.getoption('--answer_store')
+
+
+@pytest.fixture(autouse=True, scope='module')
+def test_root(request):
+    return request.config.getoption('--test_root')

acis_thermal_check/regression_testing.py

@@ -15,6 +15,8 @@
                             "MAR0817B", "MAR1117A", "APR0217B", "SEP0917C"]}
 all_loads = test_loads["normal"]+test_loads["interrupt"]
 
+nlets = {"MAR0617A", "MAR0817B", "SEP0417A"}
+
 
 class TestArgs(object):
     """
@@ -51,10 +53,14 @@ class TestArgs(object):
     model_spec : string, optional
         The path to the model specification file to use. Default is to
         use the model specification file stored in the model package.
+    nlet_file : string, optional
+        The path to an alternative NLET file to be used. Default: None,
+        which is to use the default one. 
     """
     def __init__(self, name, outdir, model_path, run_start=None,
                  load_week=None, days=21.0, T_init=None, interrupt=False,
-                 state_builder='acis', verbose=0, model_spec=None):
+                 state_builder='acis', verbose=0, model_spec=None,
+                 nlet_file=None):
         from datetime import datetime
         self.load_week = load_week
         if run_start is None:
@@ -72,7 +78,9 @@ def __init__(self, name, outdir, model_path, run_start=None,
             load_letter = load_week[-1].lower()
             self.backstop_file = "/data/acis/LoadReviews/%s/%s/ofls%s" % (load_year, load_week[:-1], load_letter)
         self.days = days
-        self.nlet_file = '/data/acis/LoadReviews/NonLoadTrackedEvents.txt'
+        if nlet_file is None:
+            nlet_file = '/data/acis/LoadReviews/NonLoadTrackedEvents.txt'
+        self.nlet_file = nlet_file
         self.interrupt = interrupt
         self.state_builder = state_builder
         self.pred_only = False
@@ -121,8 +129,8 @@ def exception_catcher(test, old, new, data_type, **kwargs):
 
 
 class RegressionTester(object):
-    def __init__(self, atc_class, model_path, model_spec, 
-                 atc_args=None, atc_kwargs=None):
+    def __init__(self, atc_class, model_path, model_spec, atc_args=None,
+                 atc_kwargs=None, test_root=None, sub_dir=None):
         self.model_path = model_path
         if atc_args is None:
             atc_args = ()
@@ -134,11 +142,16 @@ def __init__(self, atc_class, model_path, model_spec,
         self.valid_limits = self.atc_obj.validation_limits
         self.hist_limit = self.atc_obj.hist_limit
         self.curdir = os.getcwd()
-        self.tmpdir = tempfile.mkdtemp()
-        self.outdir = os.path.abspath(os.path.join(self.tmpdir, self.name+"_test"))
+        if test_root is None:
+            rootdir = tempfile.mkdtemp()
+        else:
+            rootdir = test_root
+        if sub_dir is not None:
+            rootdir = os.path.join(rootdir, sub_dir)
+        self.outdir = os.path.abspath(rootdir)
         self.test_model_spec = os.path.join(model_path, "tests", model_spec)
         if not os.path.exists(self.outdir):
-            os.mkdir(self.outdir)
+            os.makedirs(self.outdir, exist_ok=True)
 
     def run_model(self, load_week, run_start=None, state_builder='acis',
                   interrupt=False, override_limits=None):
@@ -157,14 +170,18 @@ def run_model(self, load_week, run_start=None, state_builder='acis',
             "acis", default "acis".
         interrupt : boolean, optional
             Whether or not this is an interrupt load. Default: False
-        use the model specification file stored in the model package.
-            override_limits : dict, optional
+        override_limits : dict, optional
             Override any margin by setting a new value to its name
             in this dictionary. SHOULD ONLY BE USED FOR TESTING.
         """
         out_dir = os.path.join(self.outdir, load_week)
+        if load_week in nlets:
+            nlet_file = os.path.join(os.path.dirname(__file__), 
+                                     f'data/nlets/TEST_NLET_{load_week}.txt')
+        else:
+            nlet_file = None
         args = TestArgs(self.name, out_dir, self.model_path, run_start=run_start,
-                        load_week=load_week, interrupt=interrupt,
+                        load_week=load_week, interrupt=interrupt, nlet_file=nlet_file,
                         state_builder=state_builder, model_spec=self.test_model_spec)
         self.atc_obj.run(args, override_limits=override_limits)
 

acis_thermal_check/state_builder.py

@@ -263,6 +263,10 @@ def __init__(self, interrupt=False, backstop_file=None, nlet_file=None,
             self.tstart = rev_bs_cmds[0]['time']
             self.tstop = rev_bs_cmds[-1]['time']
 
+            # Initialize the end time attribute for event searches within the BSC object
+            # At the beginning, it will be the time of the last command in the Review Load
+            self.BSC.end_event_time = rev_bs_cmds[-1]['time']
+
         # Connect to database (NEED TO USE aca_read for sybase; user is ignored for sqlite)
         # We only need this as the quick way to get the validation states.
         server = os.path.join(os.environ['SKA'], 'data', 'cmd_states', 'cmd_states.db3')
@@ -316,13 +320,18 @@ def get_prediction_states(self, tbegin):
         import copy
 
         bs_cmds = copy.copy(self.bs_cmds)
+
+        # Capture the start time of the review load
         bs_start_time = bs_cmds[0]['time']
+
+        # Capture the path to the ofls directory
         present_ofls_dir = copy.copy(self.backstop_file)
 
         # So long as the earliest command in bs_cmds is after the state0
         # time, keep concatenating continuity commands to bs_cmds based upon
         # the type of load.
-        # Note that as you march back in time along the load chain, "ofls_dir" will change.
+        # Note that as you march back in time along the load chain, 
+        # "present_ofls_dir" will change.
 
         # First we need a State0 because cmd_states.get_states cannot translate
         # backstop commands into commanded states without one. cmd_states.get_state0
@@ -347,7 +356,8 @@ def get_prediction_states(self, tbegin):
                 # Obtain the continuity load commands
                 cont_bs_cmds, cont_bs_name = self.BSC.get_bs_cmds(cont_load_path)
 
-                # Combine the continuity commands with the bs_cmds
+                # Combine the continuity commands with the bs_cmds. The result
+                # is stored in bs_cmds
                 bs_cmds = self.BSC.CombineNormal(cont_bs_cmds, bs_cmds)
 
                 # Reset the backstop collection start time for the While loop

doc/source/developing_models.rst

@@ -417,7 +417,7 @@ tests the legacy "SQL" state builder, and another which checks for violations.
 All of these scripts make use of a ``RegressionTester`` class which handles all
 of the testing. 
 
-The "ACIS" state builder script generates a ``RegressionTester`` object
+The "ACIS" state builder test script generates a ``RegressionTester`` object
 appropriate to the model to be tested, runs the models using the ``run_models``
 method, called with the appropriate state builder, and then runs prediction
 and validation tests. The ``test_dpa_acis.py`` script for the 1DPAMZT model is
@@ -433,20 +433,25 @@ use of this argument is explained in :ref:`test_suite`.
         RegressionTester, all_loads
     import pytest
 
-    dpa_rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json")
 
-    # ACIS state builder tests
-
-    dpa_rt.run_models(state_builder='acis')
+    @pytest.fixture(autouse=True, scope='module')
+    def dpa_rt(test_root):
+        # ACIS state builder tests
+        rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json",
+                              test_root=test_root, sub_dir='acis')
+        rt.run_models(state_builder='acis')
+        return rt
 
     # Prediction tests
+
     @pytest.mark.parametrize('load', all_loads)
-    def test_prediction(answer_store, load):
+    def test_prediction(dpa_rt, answer_store, load):
         dpa_rt.run_test("prediction", load, answer_store=answer_store)
 
     # Validation tests
+
     @pytest.mark.parametrize('load', all_loads)
-    def test_validation(answer_store, load):
+    def test_validation(dpa_rt, answer_store, load):
         dpa_rt.run_test("validation", load, answer_store=answer_store)
 
 The "SQL" state builder tests are nearly identical to the "ACIS" ones, but in
@@ -456,28 +461,28 @@ is a test of that. The example for the 1DPAMZT model is shown below:
 
 .. code-block:: python
 
-    from ..dpa_check import model_path, DPACheck
-    from acis_thermal_check.regression_testing import \
-        RegressionTester, all_loads
-    import pytest
-
-    dpa_rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json")
-
-    # SQL state builder tests
-
-    dpa_rt.run_models(state_builder='sql')
+    @pytest.fixture(autouse=True, scope='module')
+    def dpa_rt(test_root):
+        # ACIS state builder tests
+        rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json",
+                              test_root=test_root, sub_dir='sql')
+        rt.run_models(state_builder='sql')
+        return rt
 
     # Prediction tests
+
     @pytest.mark.parametrize('load', all_loads)
-    def test_prediction(answer_store, load):
+    def test_prediction(dpa_rt, answer_store, load):
         if not answer_store:
             dpa_rt.run_test("prediction", load)
         else:
             pass
 
     # Validation tests
+
+
     @pytest.mark.parametrize('load', all_loads)
-    def test_validation(answer_store, load):
+    def test_validation(dpa_rt, answer_store, load):
         if not answer_store:
             dpa_rt.run_test("validation", load)
         else:
@@ -513,13 +518,12 @@ like this:
     from acis_thermal_check.regression_testing import \
         RegressionTester
     import os
-
-    dpa_rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json")
-
-
-    def test_JUL3018A_viols(answer_store):
+
+    def test_JUL3018A_viols(answer_store, test_root):
         answer_data = os.path.join(os.path.dirname(__file__), "answers",
                                    "JUL3018A_viol.json")
+        dpa_rt = RegressionTester(DPACheck, model_path, "dpa_test_spec.json",
+                                  test_root=test_root, sub_dir='viols')
         dpa_rt.check_violation_reporting("JUL3018A", answer_data,
                                          answer_store=answer_store)
 

doc/source/test_suite.rst

@@ -46,6 +46,17 @@ directory (e.g., ``dpa_check``) and run ``py.test`` like so:
 
 The ``-s`` flag is optionally included here so that the output has maximum verbosity.
 
+Normally, the outputs of the thermal model runs are stored in a temporary directory
+which is discarded after the tests have been carried out. If you want to dump these
+outputs to a different location for later examination, use the ``test_root`` argument
+on the command line:
+
+.. code-block:: bash
+
+    [~]$ cd ~/Source/dpa_check
+
+    [~]$ py.test -s . --test_root=/Users/jzuhone/dpa_tests
+
 You can also import any model package from an interactive Python session and run the 
 ``test()`` method on it:
 

setup.py

@@ -3,6 +3,7 @@
 import glob
 
 templates = glob.glob("templates/*")
+data = glob.glob("data/*")
 
 setup(name='acis_thermal_check',
       packages=["acis_thermal_check"],
@@ -12,6 +13,6 @@
       author='John ZuHone',
       author_email='john.zuhone@cfa.harvard.edu',
       url='http://github.com/acisops/acis_thermal_check',
-      data_files=[('templates', templates)],
+      data_files=[('templates', templates), ('data', data)],
       include_package_data=True,
       )