Increase functionality of try_toolchain

easybuild/framework/easyconfig/easyconfig.py

@@ -115,16 +115,16 @@ def toolchain_hierarchy_cache(func):
     cache = {}
 
     @functools.wraps(func)
-    def cache_aware_func(toolchain):
+    def cache_aware_func(toolchain, require_capabilities=False):
         """Look up toolchain hierarchy in cache first, determine and cache it if not available yet."""
-        cache_key = (toolchain['name'], toolchain['version'])
+        cache_key = (toolchain['name'], toolchain['version'], require_capabilities)
 
         # fetch from cache if available, cache it if it's not
         if cache_key in cache:
             _log.debug("Using cache to return hierarchy for toolchain %s: %s", str(toolchain), cache[cache_key])
             return cache[cache_key]
         else:
-            toolchain_hierarchy = func(toolchain)
+            toolchain_hierarchy = func(toolchain, require_capabilities)
             cache[cache_key] = toolchain_hierarchy
             return cache[cache_key]
 
@@ -135,7 +135,7 @@ def cache_aware_func(toolchain):
 
 
 @toolchain_hierarchy_cache
-def get_toolchain_hierarchy(parent_toolchain):
+def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False):
     """
     Determine list of subtoolchains for specified parent toolchain.
     Result starts with the most minimal subtoolchains first, ends with specified toolchain.
@@ -234,6 +234,32 @@ def get_toolchain_hierarchy(parent_toolchain):
         subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
         toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version})
 
+        # also add toolchain capabilities
+        if require_capabilities:
+            for toolchain in toolchain_hierarchy:
+                toolchain_class, _ = search_toolchain(toolchain['name'])
+                tc = toolchain_class(version=toolchain['version'])
+                try:
+                    toolchain['compiler_family'] = tc.comp_family()
+                except EasyBuildError:
+                    toolchain['compiler_family'] = None
+                try:
+                    toolchain['mpi_family'] = tc.mpi_family()
+                except EasyBuildError:
+                    toolchain['mpi_family'] = None
+                try:
+                    toolchain['blas_family'] = tc.blas_family()
+                except EasyBuildError:
+                    toolchain['blas_family'] = None
+                try:
+                    toolchain['lapack_family'] = tc.lapack_family()
+                except EasyBuildError:
+                    toolchain['lapack_family'] = None
+                if 'CUDA_CC' in tc.variables:
+                    toolchain['cuda'] = True
+                else:
+                    toolchain['cuda'] = None  # Useful to have it consistent with the rest
+
     _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy)
     return toolchain_hierarchy
 

easybuild/framework/easyconfig/tweak.py

@@ -44,6 +44,7 @@
 
 from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default
 from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig
+from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import read_file, write_file
@@ -183,7 +184,7 @@ def __repr__(self):
         tweaks['checksums'] = []
         _log.warning("Tweaking version: checksums cleared, verification disabled.")
 
-    # we need to treat list values seperately, i.e. we prepend to the current value (if any)
+    # we need to treat list values separately, i.e. we prepend to the current value (if any)
     for (key, val) in tweaks.items():
 
         if isinstance(val, list):
@@ -615,3 +616,67 @@ def obtain_ec_for(specs, paths, fp=None):
         return res
     else:
         raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.")
+
+
+def compare_toolchain_specs(source_tc_spec, target_tc_spec):
+    """
+    Compare whether a source and target toolchain have compatible characteristics
+
+    :param source_tc_spec: specs of source toolchain
+    :param target_tc_spec: specs of target toolchain
+    """
+    can_map = True
+    # Check they have same capabilities
+    for key in ['compiler_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda']:
+        if target_tc_spec[key] is None and source_tc_spec[key] is not None:
+            can_map = False
+            break
+
+    return can_map
+
+
+def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy):
+    """
+    Match a source toolchain spec to the minimal corresponding toolchain in a target hierarchy
+
+    :param source_tc_spec: specs of source toolchain
+    :param target_tc_hierarchy: hierarchy of specs for target toolchain
+    """
+    minimal_matching_toolchain = {}
+    target_compiler_family = ''
+    # Do a complete loop so we always end up with the minimal value in the hierarchy
+    # hierarchy is given from lowest to highest, so need to reverse the order in the list
+    for target_tc_spec in reversed(target_tc_hierarchy):
+        if compare_toolchain_specs(source_tc_spec, target_tc_spec):
+            # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore
+            if (source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] != 'GCCcore')\
+                    or (source_tc_spec['name'] == 'GCCcore' and target_tc_spec['name'] == 'GCCcore'):
+                minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']}
+                target_compiler_family = target_tc_spec['compiler_family']
+
+    if not minimal_matching_toolchain:
+        raise EasyBuildError("No possible mapping from source toolchain spec %s to target toolchain hierarchy specs %s",
+                             source_tc_spec, target_tc_hierarchy)
+
+    # Warn if we are changing compiler families, this is very likely to cause problems
+    if target_compiler_family != source_tc_spec['compiler_family']:
+        print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!" %
+                      (source_tc_spec['compiler_family'], target_compiler_family))
+
+    return minimal_matching_toolchain
+
+
+def map_toolchain_hierarchies(source_toolchain, target_toolchain):
+    """
+    Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain
+
+    :param source_toolchain: initial toolchain of the easyconfig(s)
+    :param target_toolchain: target toolchain for tweaked easyconfig(s)
+    """
+    tc_mapping = {}
+    initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, require_capabilities=True)
+    target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, require_capabilities=True)
+    for toolchain_spec in initial_tc_hierarchy:
+        tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy)
+
+    return tc_mapping

test/framework/robot.py

@@ -701,6 +701,33 @@ def test_get_toolchain_hierarchy(self):
             {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
         ])
 
+        # test also --try-toolchain* case, where we want more detailed information
+        init_config(build_options={
+            'valid_module_classes': module_classes(),
+            'robot_path': test_easyconfigs,
+        })
+
+        get_toolchain_hierarchy.clear()
+
+        goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True)
+        self.assertEqual(goolf_hierarchy, [
+            {'name': 'GCC', 'version': '4.7.2', 'compiler_family': 'GCC', 'mpi_family': None,
+             'lapack_family': None, 'blas_family': None, 'cuda': None},
+            {'name': 'gompi', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI',
+             'lapack_family': None, 'blas_family': None, 'cuda': None},
+            {'name': 'goolf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI',
+             'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': None},
+        ])
+
+        iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
+                                                  require_capabilities=True)
+        self.assertEqual(iimpi_hierarchy, [
+            {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': None,
+             'lapack_family': None, 'blas_family': None, 'cuda': None},
+            {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': 'IntelMPI',
+             'lapack_family': None, 'blas_family': None, 'cuda': None},
+        ])
+
         # test also including dummy
         init_config(build_options={
             'add_dummy_to_minimal_toolchains': True,

test/framework/tweak.py

@@ -32,9 +32,13 @@
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
 
+from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy
 from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one
+from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs
+from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import module_classes
 from easybuild.tools.filetools import write_file
 
 
@@ -145,6 +149,76 @@ def test_tweak_one_version(self):
         tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
         self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
 
+    def test_compare_toolchain_specs(self):
+        """Test comparing the functionality of two toolchains"""
+        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        init_config(build_options={
+            'valid_module_classes': module_classes(),
+            'robot_path': test_easyconfigs,
+        })
+        get_toolchain_hierarchy.clear()
+        goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True)
+        iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
+                                                  require_capabilities=True)
+
+        # Hierarchies are returned with top-level toolchain last, goolf has 3 elements here, intel has 2
+        # goolf <-> iimpi (should return False)
+        self.assertFalse(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1]), "goolf requires math libs")
+        # gompi <-> iimpi
+        self.assertTrue(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1]))
+        # GCC <-> iimpi
+        self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[1]))
+        # GCC <-> iccifort
+        self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[0]))
+
+    def test_match_minimum_tc_specs(self):
+        """Test matching a toolchain to lowest possible in a hierarchy"""
+        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        init_config(build_options={
+            'valid_module_classes': module_classes(),
+            'robot_path': test_easyconfigs,
+        })
+        get_toolchain_hierarchy.clear()
+        goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True)
+        iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
+                                                  require_capabilities=True)
+
+        # Compiler first
+        self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], goolf_hierarchy),
+                         {'name': 'GCC', 'version': '4.7.2'})
+        # Then MPI
+        self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], goolf_hierarchy),
+                         {'name': 'gompi', 'version': '1.4.10'})
+        # Check against itself for math
+        self.assertEqual(match_minimum_tc_specs(goolf_hierarchy[2], goolf_hierarchy),
+                         {'name': 'goolf', 'version': '1.4.10'})
+        # Make sure there's an error when we can't do the mapping
+        error_msg = "No possible mapping from source toolchain spec .*"
+        self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs,
+                              goolf_hierarchy[2], iimpi_hierarchy)
+
+    def test_map_toolchain_hierarchies(self):
+        """Test between two toolchain hierarchies"""
+        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        init_config(build_options={
+            'valid_module_classes': module_classes(),
+            'robot_path': test_easyconfigs,
+        })
+        get_toolchain_hierarchy.clear()
+        goolf_tc = {'name': 'goolf', 'version': '1.4.10'}
+        gompi_tc = {'name': 'gompi', 'version': '1.4.10'}
+        iimpi_tc = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}
+
+        self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc),
+                         {'iccifort': {'name': 'GCC', 'version': '4.7.2'},
+                          'iimpi': {'name': 'gompi', 'version': '1.4.10'}})
+        self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc),
+                         {'GCC': {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
+                          'gompi': {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}})
+        # Expect an error when there is no possible mapping
+        error_msg = "No possible mapping from source toolchain spec .*"
+        self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies,
+                              goolf_tc, iimpi_tc)
 
 def suite():
     """ return all the tests in this file """