Issue 189: redo shelf to shelve (#193)

* rewriting shelf

Changed a bunch of names (not all) from shelf to shelve while doing the rewrite just to make it match. It makes it much nicer in understanding the mapping of databases.

Cleaned up post2shelve to use the new shelve tables that account for versions.

Still had a bit of shelf laying about that testing found. Cleaned it up and changed some of the test harness to fit the new shelve.

The bulk of the changes came when the unit tests started running. It was here that most of the type changes and the whole concept of the db changes worked itself out.

.ci/Dockerfile.ap

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/Dockerfile.cit

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/Dockerfile.dit

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/Dockerfile.ex

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/Dockerfile.os

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/Dockerfile.py

@@ -1,5 +1,5 @@
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/check_01.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/check_02.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/check_03.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/check_04.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/check_05.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.
@@ -48,11 +48,11 @@ post_state "$context" "$description" "$state"
 
 if current_state
 then
-    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -it niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/ae --ae-pkg=ae --verbose | tee compliant.rpt.txt
+    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -t niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/ae --ae-pkg=ae --verbose | tee compliant.rpt.txt
     [[ $? -eq 0 ]] && ret_ae=passed || ret_ae=failed
-    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -it niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/bae --ae-pkg=bae --verbose | tee -a compliant.rpt.txt
+    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -t niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/bae --ae-pkg=bae --verbose | tee -a compliant.rpt.txt
     [[ $? -ne 0 ]] && ret_bae=passed || ret_bae=failed
-    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -it niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/Integration/ae --ae-pkg=ae --verbose | tee -a compliant.rpt.txt
+    docker run --rm -e PYTHONPATH=${rootdir}/Python -e USERNAME="$(whoami)" -v ${rootdir}:${rootdir} -u $UID -w ${rootdir} -t niessner/cit:$(cit_version) python3 -m dawgie.tools.compliant --ae-dir=${rootdir}/Test/Integration/ae --ae-pkg=ae --verbose | tee -a compliant.rpt.txt
     [[ $? -eq 0 ]] && ret_iae=passed || ret_iae=failed
 
     if [[ $ret_ae == "passed" ]] && [[ $ret_bae == "passed" ]] && [[ $ret_iae == "passed" ]]

.ci/exercise_00.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/exercise_01.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/pypi.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/release_notes.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/step_00.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/step_99.sh

@@ -2,7 +2,7 @@
 # need to signal the parent Jenkins script of failure if any step fails
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

.ci/util.sh

@@ -1,6 +1,6 @@
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

Bash/crew.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

Bash/sdppiped.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

Bash/worker.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

Python/Coda/aws_lambda.py

@@ -1,7 +1,7 @@
 '''
 
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.

Python/Coda/build.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # COPYRIGHT:
-# Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+# Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 # U.S. Government sponsorship acknowledged.
 #
 # All rights reserved.

Python/dawgie/__init__.py

@@ -1,7 +1,7 @@
 '''
 --
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.
@@ -256,6 +256,31 @@ class Version:
     self._get_ver() and self._set_ver(). However, this should be done with
     extreme caution.
     '''
+    def __eq__(self, other):
+        return all ([self.design() == other.design(),
+                     self.implementation() == other.implementation(),
+                     self.bugfix() == other.bugfix()])
+    def __ge__(self, other):
+        if self.design() > other.design(): return True
+        if self.design() == other.design():
+            if self.implementation() > other.implementation(): return True
+            if self.implementation() == other.implementation():
+                return self.bugfix() >= other.bugfix()
+        return False
+    def __gt__(self, other): return self.__ge__ (other) and self.__ne__ (other)
+    def __le__(self, other):
+        if self.design() < other.design(): return True
+        if self.design() == other.design():
+            if self.implementation() < other.implementation(): return True
+            if self.implementation() == other.implementation():
+                return self.bugfix() <= other.bugfix()
+        return False
+    def __lt__(self, other): return self.__le__ (other) and self.__ne__ (other)
+    def __ne__(self, other):
+        return any ([self.design() != other.design(),
+                     self.implementation() != other.implementation(),
+                     self.bugfix() != other.bugfix()])
+
     def _get_ver(self)->VERSION: return self._version_
     def _set_ver(self, ver:VERSION): self._version_ = ver
     def asstring(self)->str: return '.'.join ([str(self.design()),

Python/dawgie/context.py

@@ -1,7 +1,7 @@
 '''
 --
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.
@@ -57,7 +57,7 @@ class PortOffset(enum.Enum):
     farm = 1
     frontend = 0
     log = 2
-    shelf = 3
+    shelve = 3
     pass
 
 ae_base_path = os.environ.get ('DAWGIE_AE_BASE_PATH', '/proj/src/ae')
@@ -77,14 +77,14 @@ class PortOffset(enum.Enum):
 data_stg = os.environ.get ('DAWGIE_DATA_STAGED', '/proj/data/stg')
 
 db_host = os.environ.get ('DAWGIE_DB_HOST', 'localhost')
-db_impl = os.environ.get ('DAWGIE_DB_IMPL', 'shelf')
+db_impl = os.environ.get ('DAWGIE_DB_IMPL', 'shelve')
 db_name = os.environ.get ('DAWGIE_DB_NAME', 'undefined')
 db_path = os.environ.get ('DAWGIE_DB_PATH', '/proj/data/db')
 db_post2shelve_prefix = os.environ.get ('DAWGIE_DB_POST2SHELVE_PREFIX',
                                         'undefined')
 db_rotate_path = os.environ.get ('DAWGIE_DB_ROTATE_PATH', '/proj/data/db')
 db_copy_path = os.environ.get ('DAWGIE_DB_COPY_PATH', '/tmp')
-db_port = int(os.environ.get ('DAWGIE_DB_PORT', 8080 + PortOffset.shelf.value))
+db_port = int(os.environ.get ('DAWGIE_DB_PORT', 8080 + PortOffset.shelve.value))
 db_rotate = os.environ.get('DAWGIE_DB_ROTATES', 10)
 db_lock = False
 
@@ -231,15 +231,15 @@ def override (args):
         pass
 
     if args.context_db_impl == 'post':
-        if args.context_db_port == fe_port + PortOffset.shelf.value:
+        if args.context_db_port == fe_port + PortOffset.shelve.value:
             args.context_db_port = 5432
             pass
         if args.context_db_path.find (':') < 0:
             args.context_db_path = 'username:password'
             pass
         pass
 
-    if args.context_db_impl == 'shelf':
+    if args.context_db_impl == 'shelve':
         # This db_path is the value before it is overriden
         if args.context_db_rotate_path == db_path:
             args.context_db_rotate_path = args.context_db_path

Python/dawgie/db/__init__.py

@@ -8,7 +8,7 @@
 
 --
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.
@@ -64,10 +64,12 @@ def _db_in_use():
     m = importlib.import_module ('dawgie.db.' + dawgie.context.db_impl)
     return m
 
-def _prime_keys():
+def _prime_keys()->[str]:
+    '''return all the kyes in the form runid.targetname.task.alg.sv.valname'''
     return _db_in_use()._prime_keys()
 
-def _prime_values():
+def _prime_values()->[str]:
+    '''return all of the blob names'''
     return _db_in_use()._prime_values()
 
 def add (target_name:str)->bool:

Python/dawgie/db/lockview.py

@@ -1,6 +1,6 @@
 '''
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.

Python/dawgie/db/post.py

@@ -2,7 +2,7 @@
 
 --
 COPYRIGHT:
-Copyright (c) 2015-2022, California Institute of Technology ("Caltech").
+Copyright (c) 2015-2023, California Institute of Technology ("Caltech").
 U.S. Government sponsorship acknowledged.
 
 All rights reserved.
@@ -1033,7 +1033,7 @@ def metrics()->'[dawgie.db.METRIC_DATA]':
         try:
             msv[vn] = dawgie.db.util.decode(row[7])
             svs[key] = msv
-        except FileNotFoundError: log.error ('possible database corruption because cannot fine __metric__ state vector value: %s', row[7])
+        except FileNotFoundError: log.error ('possible database corruption because cannot find __metric__ state vector value: %s', row[7])
         pass
     for key,msv in svs.items():
         cur.execute ('SELECT name FROM Target where PK = %s;', (key[2],))