Test config version (#268)

* Started implemented version variables in test config.

* Pavilion will now output current version in results.

* Set default test version to be 1.0.

* Removed trailing new line from pav version variable.

* Implemented new version subsection in test config. Currently only outputs test version.

* Implemented default value for min_pav_version.

* Test incapatibility will cause the whole suite to stop running.

* Initial unit test for version test config.

* Put min_pav_version in the results

* Implemented version info as fields in test_run object, updated accordingly.

* Fixed how I get the version info.

* Changed to a RegexElem to ensure we get a range in the expected format.

* Fixed a typ.

* Moved compatibility check to resolver.py

* Removed unit test on run command, as version check is done elsewhere.

* Fixed some whitespace issues,I created accidently.

* Removed trailing new line in pav version variable.

* Replaces a null value with a 'None' string, for result purposes.

* Initial unit tests for version compatibility checks.

* Removed unnecessary imports in unit test

* Added comment for clarification.

* Removed version keyed elem and made min_pav_version and test_verion there own thing.

* Updated the unit tests to reflect changed made to config format.

* Changed min_pav_version to compatible_pav_versions.

* Updated help text for compatible_pav_versions.

* yaml_config changes.

* Updated compatible_pav_versions update null object as a string for results reasons.

* Updated unit tests.

* Changed default test_version to 0.0

* Fixed a spelling issue.

* Updated unit test to include silence() method.

* Compatible versions can be single version, range, or wildcard.

* Changed the help text to reflect changes made.

* Made compatible_pav_version a StrElem instead of RegexElem.

* Removed compatible_pav_versions from test results, and updated unit tests.

* Reworked the version check by breaking it into different methods.

* White space fixes.

* Made a check_version_compatibility method that calls the other methods, and cleans up the load_raw_configs method.

* Renamed some methods.

* Removed get_min_str, get_max_str methods. Removed calc_max, calc_min and replaced with verify_versions.

* Removed a variable that doesn't get reused.

* Reworked how compatible_pav_versions is verified, and broken up.

* No longer allow * in the compatible_pav_version.

* Changed how pav version is obtained.

* Reworked where compatibility check occurs, updated error handling, fixed list splice issue.

* Update file_format.py

Co-authored-by: kjeverson <kjeverson@fg-fey1.lanl.gov>
Co-authored-by: Paul Ferrell <51765748+Paul-Ferrell@users.noreply.github.com>

lib/pavilion/config.py

@@ -289,7 +289,7 @@ def get_version():
             lines = file.readlines()
             for line in lines:
                 if line.startswith('RELEASE='):
-                    return line.split('=')[1]
+                    return line.split('=')[1].strip()
 
             return '<unknown>'
 

lib/pavilion/result/base.py

@@ -34,6 +34,10 @@ def get_sched_keys(test):
              "The test run name"),
     'id': (lambda test: test.id,
            "The test run id"),
+    'test_version': (lambda test: test.test_version,
+                     "The test config version."),
+    'pav_version': (lambda test: test.var_man['pav.version'],
+                    "The version of Pavilion used to run this test."),
     'created': (lambda test: datetime.datetime.fromtimestamp(
                 test.path.stat().st_mtime).isoformat(" "),
                 "When the test was created."),

lib/pavilion/test_config/file_format.py

@@ -260,6 +260,17 @@ class TestConfigLoader(yc.YamlConfigLoader):
                       "the Pavilion variable matches one or more of the "
                       " values."
         ),
+        yc.StrElem(
+            'compatible_pav_versions', default='',
+            help_text="Specify compatibile pavilion versions for this "
+                      "specific test. Can be represented as a single "
+                      "version, ex: 1, 1.2, 1.2.3, or a range, "
+                      "ex: 1.2-1.3.4, etc."
+        ),
+        yc.StrElem(
+            'test_version', default='0.0',
+            help_text="Documented test version."
+        ),
         yc.KeyedElem(
             'build', elements=[
                 yc.ListElem(

lib/pavilion/test_config/resolver.py

@@ -11,6 +11,7 @@
 import io
 import logging
 import os
+import re
 from collections import defaultdict
 from typing import List, IO, Tuple
 
@@ -19,6 +20,7 @@
 from pavilion import pavilion_variables
 from pavilion import schedulers
 from pavilion import system_variables
+from pavilion.pavilion_variables import PavVars
 from pavilion.test_config import parsers
 from pavilion.test_config import variables
 from pavilion.test_config.file_format import (TestConfigError, TEST_NAME_RE,
@@ -33,6 +35,8 @@
 
 LOGGER = logging.getLogger('pav.' + __name__)
 
+TEST_VERS_RE = re.compile(r'^\d+(\.\d+){0,2}$')
+
 
 class TestConfigResolver:
     """Converts raw test configurations into their final, fully resolved
@@ -469,6 +473,61 @@ def load_raw_configs(self, tests, host, modes):
 
         return picked_tests
 
+    def verify_version_range(comp_versions):
+
+        if comp_versions.count('-') > 1:
+            raise TestConfigError(
+                "Invalid compatible_pav_versions value ('{}'). Not a valid "
+                "range.".format(comp_versions))
+
+        min_str = comp_versions.split('-')[0]
+        max_str = comp_versions.split('-')[-1]
+
+        min_version = TestConfigResolver.verify_version(min_str, comp_versions)
+        max_version = TestConfigResolver.verify_version(max_str, comp_versions)
+
+        return min_version, max_version
+
+    def verify_version(version_str, comp_versions):
+        """Ensures version was provided in the correct format, and returns the
+        version as a list of digits."""
+
+        if TEST_VERS_RE.match(version_str) is not None:
+            version = version_str.split(".")
+            return [int(i) for i in version]
+        else:
+            raise TestConfigError(
+                "Invalid compatible_pav_versions value '{}' in '{}'. "
+                "Compatible versions must be of form X, X.X, or X.X.X ."
+                .format(version_str, comp_versions))
+
+    def check_version_compatibility(test_cfg):
+        """Returns a bool on if the test is compatible with the current version
+        of pavilion."""
+
+        version = PavVars()['version']
+        version = [int(i) for i in version.split(".")]
+        comp_versions = test_cfg.get('compatible_pav_versions')
+
+        # If no version is provided we assume compatibility
+        if not comp_versions:
+            return True
+
+        min_version, max_version = TestConfigResolver.verify_version_range(comp_versions)
+
+        # Trim pavilion version to the degree dictated by min and max version.
+        # This only matters if they are equal, and only occurs when a specific 
+        # version is provided.  
+        if min_version == max_version and len(min_version) < len(version):
+            offset = len(version) - len(min_version)
+            version = version[:-offset]
+        if min_version <= version <= max_version:
+            return True
+        else:
+            raise TestConfigError(
+                "Incompatible with pavilion version '{}', compatible versions "
+                "'{}'.".format(PavVars()['version'], comp_versions))
+
     def apply_host(self, test_cfg, host):
         """Apply the host configuration to the given config."""
 
@@ -675,6 +734,12 @@ def resolve_inheritance(base_config, suite_cfg, suite_path):
                     "Loaded test '{}' in suite '{}' raised a type error, "
                     "but that should never happen. {}"
                     .format(test_name, suite_path, err))
+            try:
+                TestConfigResolver.check_version_compatibility(test_config)
+            except TestConfigError as err:
+                raise TestConfigError(
+                   "Test '{}' in suite '{}' has incompatibility issues:\n{}"
+                   .format(test_name, suite_path, err))
 
         return suite_tests
 

lib/pavilion/test_run.py

@@ -138,6 +138,10 @@ def __init__(self, pav_cfg, config,
 
         self.id = None  # pylint: disable=invalid-name
 
+        # Get the test version information
+        self.test_version = config.get('test_version')
+        self.compatible_pav_versions = config.get('compatible_pav_versions')
+
         self._attrs = {}
 
         # Mark the run to build locally.

test/data/pav_config_dir/tests/version_compatible.yaml

@@ -0,0 +1,16 @@
+one:
+    test_version: 1.2.3
+    run:
+        cmds:
+            - 'sleep 1'
+two:
+    test_version: beta
+    compatible_pav_versions: 1.2.3-5.4.9
+    run:
+        cmds:
+            - 'sleep 1'
+
+three:
+    run:
+        cmds:
+            - 'sleep 1'

test/data/pav_config_dir/tests/version_incompatible.yaml

@@ -0,0 +1,6 @@
+one:
+    test_version: 1.4
+    compatible_pav_versions: 1.2.4-1.2.8
+    run:
+        cmd:
+            - 'sleep 1'

test/tests/resolver_tests.py

@@ -1,9 +1,14 @@
 """Test the various components of the test resolver."""
 
 import copy
+import io
+import json
 
+from pavilion import arguments
+from pavilion import commands
 from pavilion import plugins
 from pavilion import system_variables
+from pavilion.pavilion_variables import PavVars
 from pavilion.test_config import TestConfigError, resolver
 from pavilion.test_config import variables
 from pavilion.unittest import PavTestCase
@@ -466,3 +471,60 @@ def test_env_order(self):
                              "Got the following instead: \n{}"
                              .format(''.join(["{}: {}\n".format(*v) for v in
                                               exports])))
+
+    def test_version_compatibility(self):
+        """Make sure version compatibility checks are working and populate the
+        results.json file correctly."""
+
+        pav_version = PavVars.version(self)
+
+        arg_parser = arguments.get_parser()
+        args = arg_parser.parse_args([
+            'run',
+            '-w',
+            '5',
+            'version_compatible'
+        ])
+
+        expected_results = {
+            'version_compatible.one': {
+                'test_version': '1.2.3',
+                'pav_version': pav_version
+            },
+            'version_compatible.two': {
+                'test_version': 'beta',
+                'pav_version': pav_version
+            },
+            'version_compatible.three': {
+                'test_version': '0.0',
+                'pav_version': pav_version
+            }
+        }
+
+        # Ensures Version information gets populated correclty even with empty
+        # version section in test config
+        run_cmd = commands.get_command(args.command_name)
+        run_cmd.silence()
+        run_cmd.run(self.pav_cfg, args)
+
+        for test in run_cmd.last_tests:
+            results = test.load_results()
+            name = results['name']
+            for key in expected_results[name].keys():
+                self.assertEqual(expected_results[name][key],
+                                 results[key])
+
+    def test_version_incompatibility(self):
+        """Make sure incompatible versions exit gracefully when attempting to
+        run."""
+
+        arg_parser = arguments.get_parser()
+        args = arg_parser.parse_args([
+            'run',
+            'version_incompatible'
+        ])
+
+        run_cmd = commands.get_command(args.command_name)
+        run_cmd.outfile = io.StringIO()
+        run_cmd.errfile = run_cmd.outfile
+        self.assertEqual(run_cmd.run(self.pav_cfg, args), 22)