From d5ba7d6cc5328afc1766fe1c7f3e041b747a8d02 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Jul 2018 14:21:26 +0200 Subject: [PATCH 01/51] Add support for including toolchains capabilities when creating toolchains hierarchy --- easybuild/framework/easyconfig/easyconfig.py | 27 ++++++++++++++++++++ test/framework/robot.py | 27 ++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index fa51ce812b..2d66bd791b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -234,6 +234,33 @@ def get_toolchain_hierarchy(parent_toolchain): subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version}) + # also add toolchain capabilities if we are using try_toolchain_* + build_specs = build_option('build_specs') + if build_specs is not None and ('toolchain_name' in build_specs or 'toolchain_name' in build_specs): + for toolchain in toolchain_hierarchy: + toolchain_class, _ = search_toolchain(toolchain['name']) + tc = toolchain_class(version=toolchain['version']) + try: + toolchain['compiler_family'] = tc.comp_family() + except EasyBuildError: + toolchain['compiler_family'] = None + try: + toolchain['mpi_family'] = tc.mpi_family() + except: + toolchain['mpi_family'] = None + try: + toolchain['blas_family'] = tc.blas_family() + except: + toolchain['blas_family'] = None + try: + toolchain['lapack_family'] = tc.lapack_family() + except: + toolchain['lapack_family'] = None + if 'CUDA_CC' in tc.variables: + toolchain['cuda'] = True + else: + toolchain['cuda'] = False + _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy diff --git a/test/framework/robot.py b/test/framework/robot.py index 8512e6f709..46a7b683da 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -701,6 +701,33 @@ def test_get_toolchain_hierarchy(self): {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, ]) + # test also including dummy + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + 'build_specs': {'toolchain_name': 'fake'}, + }) + + get_toolchain_hierarchy.clear() + + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}) + self.assertEqual(goolf_hierarchy, [ + {'name': 'GCC', 'version': '4.7.2', 'compiler_family': 'GCC', 'mpi_family': None, + 'lapack_family': None, 'blas_family': None, 'cuda': False}, + {'name': 'gompi', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', + 'lapack_family': None, 'blas_family': None, 'cuda': False}, + {'name': 'goolf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', + 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': False}, + ]) + + iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}) + self.assertEqual(iimpi_hierarchy, [ + {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': None, + 'lapack_family': None, 'blas_family': None, 'cuda': False}, + {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': 'IntelMPI', + 'lapack_family': None, 'blas_family': None, 'cuda': False}, + ]) + # test also including dummy init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, From cb4904d5814f5bef03216c9f86be16abdd21073f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Jul 2018 14:29:58 +0200 Subject: [PATCH 02/51] Update comment in tests --- test/framework/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index 46a7b683da..c6de47233b 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -701,7 +701,7 @@ def test_get_toolchain_hierarchy(self): {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, ]) - # test also including dummy + # test also --try-toolchain* case, where we want more detailed information init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, From b0c540632598fc831cd917d3e10b8018a0fd40b5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Jul 2018 14:32:16 +0200 Subject: [PATCH 03/51] No bare excepts --- easybuild/framework/easyconfig/easyconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2d66bd791b..066965425e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -246,15 +246,15 @@ def get_toolchain_hierarchy(parent_toolchain): toolchain['compiler_family'] = None try: toolchain['mpi_family'] = tc.mpi_family() - except: + except EasyBuildError: toolchain['mpi_family'] = None try: toolchain['blas_family'] = tc.blas_family() - except: + except EasyBuildError: toolchain['blas_family'] = None try: toolchain['lapack_family'] = tc.lapack_family() - except: + except EasyBuildError: toolchain['lapack_family'] = None if 'CUDA_CC' in tc.variables: toolchain['cuda'] = True From 5259b4a76734ae1ae53de7c080b54c2325e9d406 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Jul 2018 18:40:37 +0200 Subject: [PATCH 04/51] No bare excepts --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/tweak.py | 60 +++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 066965425e..10184f9f60 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -259,7 +259,7 @@ def get_toolchain_hierarchy(parent_toolchain): if 'CUDA_CC' in tc.variables: toolchain['cuda'] = True else: - toolchain['cuda'] = False + toolchain['cuda'] = None # Useful to have it consistent with the rest _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ed3fc0362f..29377905f6 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -44,6 +44,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file @@ -183,7 +184,7 @@ def __repr__(self): tweaks['checksums'] = [] _log.warning("Tweaking version: checksums cleared, verification disabled.") - # we need to treat list values seperately, i.e. we prepend to the current value (if any) + # we need to treat list values separately, i.e. we prepend to the current value (if any) for (key, val) in tweaks.items(): if isinstance(val, list): @@ -615,3 +616,60 @@ def obtain_ec_for(specs, paths, fp=None): return res else: raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.") + +def compare_toolchain_specs(source_tc_spec, target_tc_spec): + """ + Compare whether a source and target toolchain have compatible characteristics + + :param source_tc_spec: specs of source toolchain + :param target_tc_spec: specs of target toolchain + """ + can_map = True + # Check they have same capabilities + for key in ['compiler_family', 'mpi_family','blas_family', 'lapack_family', 'cuda']: + if source_tc_spec[key] is not None and target_tc_spec[key] is None: + can_map = False + break + + return can_map + +def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): + """ + Match a source toolchain spec to the minimal corresponding toolchain in a target hierarchy + + :param source_tc_spec: specs of source toolchain + :param target_tc_hierarchy: hierarchy of specs for target toolchain + """ + minimal_matching_toolchain = {} + # Do a complete loop so we always end up with the minimal value in the hierarchy + for target_tc_spec in target_tc_hierarchy: + if compare_toolchain_specs(source_tc_spec, target_tc_spec): + minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} + target_compiler_family = target_tc_spec['compiler_family'] + + # Warn if we are changing compiler families, this is very likely to cause problems + if target_compiler_family != source_tc_spec['compiler_family']: + print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!", + source_tc_spec['compiler_family'], target_compiler_family) + + if not minimal_matching_toolchain: + EasyBuildError("No possible mapping from source toolchain spec %s and target toolchain hierarchy specs by %s", + source_tc_spec, target_tc_hierarchy) + + return minimal_matching_toolchain + + +def map_toolchain_hierarchies(source_toolchain, target_toolchain): + """ + Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain + + :param source_toolchain: initial toolchain of the easyconfig(s) + :param target_toolchain: target toolchain for tweaked easyconfig(s) + """ + tc_mapping = {} + initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain) + target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain) + for toolchain_spec in initial_tc_hierarchy: + tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) + + return tc_mapping \ No newline at end of file From c7ddafe055f9f2ae164fc1704dd15ee7bfc80fbe Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 10:32:43 +0200 Subject: [PATCH 05/51] Expand scope of get_toolchain_hierarchy without relying on build_options --- easybuild/framework/easyconfig/easyconfig.py | 13 ++++++------- easybuild/framework/easyconfig/tweak.py | 20 ++++++++++++-------- test/framework/robot.py | 16 ++++++++-------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 10184f9f60..5effbefe71 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -115,16 +115,16 @@ def toolchain_hierarchy_cache(func): cache = {} @functools.wraps(func) - def cache_aware_func(toolchain): + def cache_aware_func(toolchain, require_capabilities=False): """Look up toolchain hierarchy in cache first, determine and cache it if not available yet.""" - cache_key = (toolchain['name'], toolchain['version']) + cache_key = (toolchain['name'], toolchain['version'], require_capabilities) # fetch from cache if available, cache it if it's not if cache_key in cache: _log.debug("Using cache to return hierarchy for toolchain %s: %s", str(toolchain), cache[cache_key]) return cache[cache_key] else: - toolchain_hierarchy = func(toolchain) + toolchain_hierarchy = func(toolchain, require_capabilities) cache[cache_key] = toolchain_hierarchy return cache[cache_key] @@ -135,7 +135,7 @@ def cache_aware_func(toolchain): @toolchain_hierarchy_cache -def get_toolchain_hierarchy(parent_toolchain): +def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. @@ -234,9 +234,8 @@ def get_toolchain_hierarchy(parent_toolchain): subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version}) - # also add toolchain capabilities if we are using try_toolchain_* - build_specs = build_option('build_specs') - if build_specs is not None and ('toolchain_name' in build_specs or 'toolchain_name' in build_specs): + # also add toolchain capabilities + if require_capabilities: for toolchain in toolchain_hierarchy: toolchain_class, _ = search_toolchain(toolchain['name']) tc = toolchain_class(version=toolchain['version']) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 29377905f6..f83f8438a2 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -627,7 +627,7 @@ def compare_toolchain_specs(source_tc_spec, target_tc_spec): can_map = True # Check they have same capabilities for key in ['compiler_family', 'mpi_family','blas_family', 'lapack_family', 'cuda']: - if source_tc_spec[key] is not None and target_tc_spec[key] is None: + if target_tc_spec[key] is None and source_tc_spec[key] is not None: can_map = False break @@ -644,17 +644,21 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): # Do a complete loop so we always end up with the minimal value in the hierarchy for target_tc_spec in target_tc_hierarchy: if compare_toolchain_specs(source_tc_spec, target_tc_spec): - minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} - target_compiler_family = target_tc_spec['compiler_family'] + # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore + if source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] == 'GCCcore': + minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} + target_compiler_family = target_tc_spec['compiler_family'] + + + if not minimal_matching_toolchain: + EasyBuildError("No possible mapping from source toolchain spec %s and target toolchain hierarchy specs %s", + source_tc_spec, target_tc_hierarchy) # Warn if we are changing compiler families, this is very likely to cause problems if target_compiler_family != source_tc_spec['compiler_family']: print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!", source_tc_spec['compiler_family'], target_compiler_family) - if not minimal_matching_toolchain: - EasyBuildError("No possible mapping from source toolchain spec %s and target toolchain hierarchy specs by %s", - source_tc_spec, target_tc_hierarchy) return minimal_matching_toolchain @@ -667,8 +671,8 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain): :param target_toolchain: target toolchain for tweaked easyconfig(s) """ tc_mapping = {} - initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain) - target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain) + initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, require_capabilities=True) + target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, require_capabilities=True) for toolchain_spec in initial_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) diff --git a/test/framework/robot.py b/test/framework/robot.py index c6de47233b..bff370c059 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -705,27 +705,27 @@ def test_get_toolchain_hierarchy(self): init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, - 'build_specs': {'toolchain_name': 'fake'}, }) get_toolchain_hierarchy.clear() - goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}) + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) self.assertEqual(goolf_hierarchy, [ {'name': 'GCC', 'version': '4.7.2', 'compiler_family': 'GCC', 'mpi_family': None, - 'lapack_family': None, 'blas_family': None, 'cuda': False}, + 'lapack_family': None, 'blas_family': None, 'cuda': None}, {'name': 'gompi', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', - 'lapack_family': None, 'blas_family': None, 'cuda': False}, + 'lapack_family': None, 'blas_family': None, 'cuda': None}, {'name': 'goolf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', - 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': False}, + 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': None}, ]) - iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}) + iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, + require_capabilities=True) self.assertEqual(iimpi_hierarchy, [ {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': None, - 'lapack_family': None, 'blas_family': None, 'cuda': False}, + 'lapack_family': None, 'blas_family': None, 'cuda': None}, {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': 'IntelMPI', - 'lapack_family': None, 'blas_family': None, 'cuda': False}, + 'lapack_family': None, 'blas_family': None, 'cuda': None}, ]) # test also including dummy From 21b3a53308c6f4edae3eae3fd9596ed628018aef Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 11:33:21 +0200 Subject: [PATCH 06/51] Address style errors --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/tweak.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 5effbefe71..682d699280 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -258,7 +258,7 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): if 'CUDA_CC' in tc.variables: toolchain['cuda'] = True else: - toolchain['cuda'] = None # Useful to have it consistent with the rest + toolchain['cuda'] = None # Useful to have it consistent with the rest _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f83f8438a2..6f3e3c5b46 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -617,6 +617,7 @@ def obtain_ec_for(specs, paths, fp=None): else: raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.") + def compare_toolchain_specs(source_tc_spec, target_tc_spec): """ Compare whether a source and target toolchain have compatible characteristics @@ -626,13 +627,14 @@ def compare_toolchain_specs(source_tc_spec, target_tc_spec): """ can_map = True # Check they have same capabilities - for key in ['compiler_family', 'mpi_family','blas_family', 'lapack_family', 'cuda']: + for key in ['compiler_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda']: if target_tc_spec[key] is None and source_tc_spec[key] is not None: can_map = False break return can_map + def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): """ Match a source toolchain spec to the minimal corresponding toolchain in a target hierarchy @@ -649,7 +651,6 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['compiler_family'] - if not minimal_matching_toolchain: EasyBuildError("No possible mapping from source toolchain spec %s and target toolchain hierarchy specs %s", source_tc_spec, target_tc_hierarchy) @@ -659,7 +660,6 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!", source_tc_spec['compiler_family'], target_compiler_family) - return minimal_matching_toolchain @@ -676,4 +676,4 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain): for toolchain_spec in initial_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) - return tc_mapping \ No newline at end of file + return tc_mapping From d3aba8945744baf1ea198e106c76592827ca153c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 15:41:27 +0200 Subject: [PATCH 07/51] Add tests for hierarchy to heirarchy mapping --- easybuild/framework/easyconfig/tweak.py | 17 +++--- test/framework/tweak.py | 74 +++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6f3e3c5b46..8091fce956 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -643,22 +643,25 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): :param target_tc_hierarchy: hierarchy of specs for target toolchain """ minimal_matching_toolchain = {} + target_compiler_family = '' # Do a complete loop so we always end up with the minimal value in the hierarchy - for target_tc_spec in target_tc_hierarchy: + # hierarchy is given from lowest to highest, so need to reverse the order in the list + for target_tc_spec in reversed(target_tc_hierarchy): if compare_toolchain_specs(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore - if source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] == 'GCCcore': + if source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] != 'GCCcore' or \ + source_tc_spec['name'] == 'GCCcore' and target_tc_spec['name'] == 'GCCcore': minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['compiler_family'] - if not minimal_matching_toolchain: - EasyBuildError("No possible mapping from source toolchain spec %s and target toolchain hierarchy specs %s", - source_tc_spec, target_tc_hierarchy) + if len(minimal_matching_toolchain) == 0: + raise EasyBuildError("No possible mapping from source toolchain spec %s to target toolchain hierarchy specs %s", + source_tc_spec, target_tc_hierarchy) # Warn if we are changing compiler families, this is very likely to cause problems if target_compiler_family != source_tc_spec['compiler_family']: - print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!", - source_tc_spec['compiler_family'], target_compiler_family) + print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!" % + (source_tc_spec['compiler_family'], target_compiler_family)) return minimal_matching_toolchain diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 022e7a4a97..0bf99defea 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -32,9 +32,13 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one +from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs +from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import module_classes from easybuild.tools.filetools import write_file @@ -145,6 +149,76 @@ def test_tweak_one_version(self): tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') + def test_compare_toolchain_specs(self): + """Test comparing the functionality of two toolchains""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + }) + get_toolchain_hierarchy.clear() + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) + iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, + require_capabilities=True) + + # Hierarchies are returned with top-level toolchain last, goolf has 3 elements here, intel has 2 + # goolf <-> iimpi (should return False) + self.assertFalse(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1]), "goolf requires math libs") + # gompi <-> iimpi + self.assertTrue(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1])) + # GCC <-> iimpi + self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[1])) + # GCC <-> iccifort + self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[0])) + + def test_match_minimum_tc_specs(self): + """Test matching a toolchain to lowest possible in a hierarchy""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + }) + get_toolchain_hierarchy.clear() + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) + iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, + require_capabilities=True) + + # Compiler first + self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], goolf_hierarchy), + {'name': 'GCC', 'version': '4.7.2'}) + # Then MPI + self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], goolf_hierarchy), + {'name': 'gompi', 'version': '1.4.10'}) + # Check against itself for math + self.assertEqual(match_minimum_tc_specs(goolf_hierarchy[2], goolf_hierarchy), + {'name': 'goolf', 'version': '1.4.10'}) + # Make sure there's an error when we can't do the mapping + error_msg = "No possible mapping from source toolchain spec .*" + self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs, + goolf_hierarchy[2], iimpi_hierarchy) + + def test_map_toolchain_hierarchies(self): + """Test between two toolchain hierarchies""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + }) + get_toolchain_hierarchy.clear() + goolf_tc = {'name': 'goolf', 'version': '1.4.10'} + gompi_tc = {'name': 'gompi', 'version': '1.4.10'} + iimpi_tc = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} + + self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc), + {'iccifort': {'name': 'GCC', 'version': '4.7.2'}, + 'iimpi': {'name': 'gompi', 'version': '1.4.10'}}) + self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc), + {'GCC': {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, + 'gompi': {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}}) + # Expect an error when there is no possible mapping + error_msg = "No possible mapping from source toolchain spec .*" + self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies, + goolf_tc, iimpi_tc) def suite(): """ return all the tests in this file """ From 46714a9b72e32948f6370f7c8f59f8acc982e1d9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 15:50:38 +0200 Subject: [PATCH 08/51] Be conservative with logic --- easybuild/framework/easyconfig/tweak.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 8091fce956..4b1f358992 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -649,12 +649,12 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): for target_tc_spec in reversed(target_tc_hierarchy): if compare_toolchain_specs(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore - if source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] != 'GCCcore' or \ - source_tc_spec['name'] == 'GCCcore' and target_tc_spec['name'] == 'GCCcore': + if (source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] != 'GCCcore')\ + or (source_tc_spec['name'] == 'GCCcore' and target_tc_spec['name'] == 'GCCcore'): minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['compiler_family'] - if len(minimal_matching_toolchain) == 0: + if not minimal_matching_toolchain: raise EasyBuildError("No possible mapping from source toolchain spec %s to target toolchain hierarchy specs %s", source_tc_spec, target_tc_hierarchy) From 4a86c2b0d5016034f77b102784eef9db3d9a4cf4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 16:19:01 +0200 Subject: [PATCH 09/51] Remove hardcoded GCCcore name --- easybuild/framework/easyconfig/tweak.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 4b1f358992..b4e825311a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -45,6 +45,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy +from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file @@ -649,8 +650,8 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): for target_tc_spec in reversed(target_tc_hierarchy): if compare_toolchain_specs(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore - if (source_tc_spec['name'] != 'GCCcore' and target_tc_spec['name'] != 'GCCcore')\ - or (source_tc_spec['name'] == 'GCCcore' and target_tc_spec['name'] == 'GCCcore'): + if (source_tc_spec['name'] != GCCcore.NAME and target_tc_spec['name'] != GCCcore.NAME)\ + or (source_tc_spec['name'] == GCCcore.NAME and target_tc_spec['name'] == GCCcore.NAME): minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['compiler_family'] From a5f1d360a5b6ff2a71bdfd6296047d41a3b82c67 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 26 Jul 2018 16:46:16 +0200 Subject: [PATCH 10/51] Allowing mapping up from GCCcore to another compiler (in the case where GCCcore does not appear in the target) --- easybuild/framework/easyconfig/tweak.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index b4e825311a..824a00d571 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -649,9 +649,10 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): # hierarchy is given from lowest to highest, so need to reverse the order in the list for target_tc_spec in reversed(target_tc_hierarchy): if compare_toolchain_specs(source_tc_spec, target_tc_spec): - # GCCcore has compiler capabilities but should only be used if the original toolchain was also GCCcore - if (source_tc_spec['name'] != GCCcore.NAME and target_tc_spec['name'] != GCCcore.NAME)\ - or (source_tc_spec['name'] == GCCcore.NAME and target_tc_spec['name'] == GCCcore.NAME): + # GCCcore has compiler capabilities but should only be used in the target if the original toolchain was also + # GCCcore + if target_tc_spec['name'] != GCCcore.NAME or \ + (source_tc_spec['name'] == GCCcore.NAME and target_tc_spec['name'] == GCCcore.NAME): minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['compiler_family'] From 69080d323458ab5b3719c7d6fc3f7f7e440d0cea Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Jul 2018 18:30:54 +0200 Subject: [PATCH 11/51] Allowing mapping up from GCCcore to another compiler (in the case where GCCcore does not appear in the target) --- easybuild/framework/easyconfig/tweak.py | 71 +++++++++++++++++++------ 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 824a00d571..1803128e00 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -83,6 +83,11 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): if len(toolchains) > 1: raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", toolchains) + # Toolchain is unique, let's store it + source_toolchain = easyconfigs[-1]['ec']['toolchain'] + modifying_toolchain = False + target_toolchain = {} + src_to_dst_tc_mapping = {} if 'name' in build_specs or 'version' in build_specs: # no recursion if software name/version build specification are included @@ -90,24 +95,39 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): orig_ecs = easyconfigs _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) else: - # build specifications should be applied to the whole dependency graph - # obtain full dependency graph for specified easyconfigs - # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first + # We're doing something with the toolchain, build specifications should be applied to the whole dependency graph + # Obtain full dependency graph for specified easyconfigs + # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) + modifying_toolchain = True + keys = build_specs.keys() + if 'toolchain_name' in keys: + target_toolchain['name'] = build_specs['toolchain_name'] + else: + target_toolchain['name'] = source_toolchain['name'] + if 'toolchain_version' in keys: + target_toolchain['version'] = build_specs['toolchain_version'] + else: + target_toolchain['version'] = source_toolchain['version'] + + src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain) + _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) - # keep track of originally listed easyconfigs (via their path) - listed_ec_paths = [ec['spec'] for ec in easyconfigs] + # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) + # We can leave any dependencies they may have as they will only be used if required (or originally listed) + _log.debug("Filtering out toolchain hierarchy for %s" % source_toolchain) - # determine toolchain based on last easyconfigs - if orig_ecs: - toolchain = orig_ecs[-1]['ec']['toolchain'] - _log.debug("Filtering using toolchain %s" % toolchain) + i = 0 + while i < len(orig_ecs): + if orig_ecs[i]['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: + # drop elements in toolchain hierarchy + del orig_ecs[i] + else: + i += 1 - # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies - if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: - while orig_ecs[0]['ec']['toolchain'] != toolchain: - orig_ecs = orig_ecs[1:] + # keep track of originally listed easyconfigs (via their path) + listed_ec_paths = [ec['spec'] for ec in easyconfigs] # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] @@ -117,12 +137,20 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path if orig_ec['spec'] in listed_ec_paths: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) + if modifying_toolchain: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + tweaked_ecs_path) + else: + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) tweaked_easyconfigs.extend(new_ecs) else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) + if modifying_toolchain: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + tweaked_ecs_deps_path) + else: + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) return tweaked_easyconfigs @@ -681,4 +709,17 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain): for toolchain_spec in initial_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) + # TODO Include binutils version mapping for things built with GCCcore + return tc_mapping + + +def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir): + """ + Take an easyconfig spec, parse it, map it to a target toolchain and dump it out + + :param ec_spec: + :param toolchain_mapping: + :param target_dir: + :return mapped_spec: + """ From 7867ba94e2ac8b68597a0a165231d499f632c0b7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 13:14:47 +0200 Subject: [PATCH 12/51] Also grab binutils version built with GCCcore --- easybuild/framework/easyconfig/tweak.py | 62 ++++++++++++++++--- .../test_ecs/g/GCC/GCC-4.9.3-2.25.eb | 2 +- .../i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb | 2 +- .../ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb | 2 +- test/framework/tweak.py | 14 ++++- 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 1803128e00..ef01e5eac4 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -50,7 +50,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.robot import resolve_dependencies +from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.utilities import quote_str @@ -109,7 +109,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): else: target_toolchain['version'] = source_toolchain['version'] - src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain) + src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) @@ -148,7 +148,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Place all tweaked dependency easyconfigs in the directory appended to the robot path if modifying_toolchain: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_deps_path) + targetdir=tweaked_ecs_deps_path) else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) @@ -695,8 +695,22 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): return minimal_matching_toolchain +def get_dep_tree_of_toolchain(toolchain_spec, modtool): + """ + Ge the dependency tree of a toolchain -def map_toolchain_hierarchies(source_toolchain, target_toolchain): + :param toolchain_spec: toolchain spec to get the dependencies of + :return: The dependency tree of the toolchain spec + """ + path = robot_find_easyconfig(toolchain_spec['name'], toolchain_spec['version']) + if path is None: + raise EasyBuildError("Could not find easyconfig for %s toolchain version %s", + toolchain_spec['name'], toolchain_spec['version']) + ec = process_easyconfig(path, validate=False) + + return resolve_dependencies(ec, modtool) + +def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): """ Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain @@ -706,20 +720,52 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain): tc_mapping = {} initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, require_capabilities=True) target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, require_capabilities=True) + for toolchain_spec in initial_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) - # TODO Include binutils version mapping for things built with GCCcore + # Check for presence of binutils in source and target toolchain dependency trees (only do this when GCCcore is + # present in both and GCCcore is not the top of the tree) + if GCCcore.NAME in [tc_spec['name'] for tc_spec in initial_tc_hierarchy]\ + and GCCcore.NAME in [tc_spec['name'] for tc_spec in target_tc_hierarchy]\ + and initial_tc_hierarchy[-1]['name'] != GCCcore.NAME: + + binutils = 'binutils' + # Determine the dependency trees + source_dep_tree = get_dep_tree_of_toolchain(initial_tc_hierarchy[-1], modtool) + target_dep_tree = get_dep_tree_of_toolchain(target_tc_hierarchy[-1], modtool) + # Find the binutils mapping + if binutils in [dep['ec']['name'] for dep in source_dep_tree]: + # We need the binutils that was built using GCCcore (we assume that everything is using standard behaviour: + # build binutils with GCCcore and then use that for anything built with GCCcore) + binutils_list = [{'version': dep['ec']['version'], 'versionsuffix': dep['ec']['versionsuffix']} + for dep in target_dep_tree if (dep['ec']['name'] == binutils) and + (dep['ec']['toolchain']['name'] == GCCcore.NAME)] + # There should be one element in this list + if len(binutils_list) != 1: + raise EasyBuildError("Target hierarchy %s should have binutils using GCCcore, can't determine mapping!" + % target_tc_hierarchy[-1]) + else: + tc_mapping[binutils] = binutils_list[0] return tc_mapping -def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir): +def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir=None): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out - :param ec_spec: - :param toolchain_mapping: + :param ec_spec: Location of original easyconfig + :param toolchain_mapping: Mapping between toolchain and target toolchain and target toolchain :param target_dir: :return mapped_spec: """ + # TODO Parse the original easyconfig + + # TODO Replace the toolchain + + # TODO Replace the toolchains of all the dependencies + + # TODO Replace the binutils version (if necessary) + + # TODO Determine the name of the modified easyconfig and dump it target_dir \ No newline at end of file diff --git a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb index 167a4d0765..86888f1a69 100644 --- a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb @@ -17,7 +17,7 @@ dependencies = [ ('GCCcore', version), # binutils built on top of GCCcore, which was built on top of (dummy-built) binutils # excluded, good enough for testing purposes - # ('binutils', binutilsver, '', ('GCCcore', version)), + ('binutils', binutilsver, '', ('GCCcore', version)), ] altroot = 'GCCcore' diff --git a/test/framework/easyconfigs/test_ecs/i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb index 12359513d8..b41c580268 100644 --- a/test/framework/easyconfigs/test_ecs/i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb @@ -19,7 +19,7 @@ versionsuffix = '-GCC-%s-%s' % (gccver, binutilsver) dependencies = [ ('GCCcore', gccver), - # ('binutils', binutilsver, '', ('GCCcore', gccver)), + ('binutils', binutilsver, '', ('GCCcore', gccver)), ] # full list of components can be obtained from pset/mediaconfig.xml in unpacked sources diff --git a/test/framework/easyconfigs/test_ecs/i/ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/i/ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb index d291b2168f..311c34d164 100644 --- a/test/framework/easyconfigs/test_ecs/i/ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/i/ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb @@ -19,7 +19,7 @@ versionsuffix = '-GCC-%s-%s' % (gccver, binutilsver) dependencies = [ ('GCCcore', gccver), - # ('binutils', binutilsver, '', ('GCCcore', gccver)), + ('binutils', binutilsver, '', ('GCCcore', gccver)), ] # full list of components can be obtained from pset/mediaconfig.xml in unpacked sources diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 0bf99defea..e6adbadb4e 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -209,16 +209,24 @@ def test_map_toolchain_hierarchies(self): gompi_tc = {'name': 'gompi', 'version': '1.4.10'} iimpi_tc = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} - self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc), + self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc, self.modtool), {'iccifort': {'name': 'GCC', 'version': '4.7.2'}, 'iimpi': {'name': 'gompi', 'version': '1.4.10'}}) - self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc), + self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), {'GCC': {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, 'gompi': {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}}) # Expect an error when there is no possible mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies, - goolf_tc, iimpi_tc) + goolf_tc, iimpi_tc, self.modtool) + + # Test that we correctly include GCCcore binutils when it is there + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.25'} + iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), + {'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, + 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, + 'binutils': {'version': '2.25', 'versionsuffix': ''}}) def suite(): """ return all the tests in this file """ From 444aea5894bb127984d6f5e184b280d21c6f7bc3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 13:16:43 +0200 Subject: [PATCH 13/51] Style fixes --- easybuild/framework/easyconfig/tweak.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ef01e5eac4..5aa0e5cf25 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -695,9 +695,10 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): return minimal_matching_toolchain + def get_dep_tree_of_toolchain(toolchain_spec, modtool): """ - Ge the dependency tree of a toolchain + Get the dependency tree of a toolchain :param toolchain_spec: toolchain spec to get the dependencies of :return: The dependency tree of the toolchain spec @@ -710,6 +711,7 @@ def get_dep_tree_of_toolchain(toolchain_spec, modtool): return resolve_dependencies(ec, modtool) + def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): """ Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain @@ -768,4 +770,4 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir # TODO Replace the binutils version (if necessary) - # TODO Determine the name of the modified easyconfig and dump it target_dir \ No newline at end of file + # TODO Determine the name of the modified easyconfig and dump it target_dir From f256a32db396321894405cd6223c2687bf08d43a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 13:21:11 +0200 Subject: [PATCH 14/51] Add missing easyconfig to test suite --- .../b/binutils/binutils-2.25-GCCcore-4.9.3.eb | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb new file mode 100644 index 0000000000..47ffbc2e76 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb @@ -0,0 +1,26 @@ +# should be software specific, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'binutils' +version = '2.25' + +homepage = 'http://directory.fsf.org/project/binutils/' +description = "binutils: GNU binary utilities" + +toolchain = {'name': 'GCCcore', 'version': '4.9.3'} + +sources = [SOURCE_TAR_GZ] +source_urls = [GNU_SOURCE] + +# Testing purposes only so remove deps +#builddependencies = [ +# ('M4', '1.4.17'), +# ('flex', '2.5.39'), +# ('Bison', '3.0.4'), +# # zlib required, but being linked in statically, so not a runtime dep +# ('zlib', '1.2.8'), +# # use same binutils version that was used when building GCC toolchain, to 'bootstrap' this binutils +# ('binutils', version, '', True) +#] + +moduleclass = 'tools' From 619c9ff424adfd48878ad3f4139135d51f60095d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 14:27:39 +0200 Subject: [PATCH 15/51] Temporarily disable alternate method (to allow existing tests to pass) --- easybuild/framework/easyconfig/tweak.py | 36 ++++++++++++++++++------- test/framework/tweak.py | 16 +++++++++-- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 5aa0e5cf25..5f6c94fb6a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -136,6 +136,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # prepended path so that they are found first). # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path + + modifying_toolchain = False # TODO Remove this to enable all tests if orig_ec['spec'] in listed_ec_paths: if modifying_toolchain: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, @@ -762,12 +764,28 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir :param target_dir: :return mapped_spec: """ - # TODO Parse the original easyconfig - - # TODO Replace the toolchain - - # TODO Replace the toolchains of all the dependencies - - # TODO Replace the binutils version (if necessary) - - # TODO Determine the name of the modified easyconfig and dump it target_dir + # Fully parse the original easyconfig + parsed_ec = process_easyconfig(ec_spec, validate=False)[0] + # Replace the toolchain + parsed_ec['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] + # Replace the toolchains of all the dependencies + for dep in parsed_ec.dependencies(): + # skip dependencies that are marked as external modules + if dep['external_module']: + continue + if dep['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] + # Replace the binutils version (if necessary) + if (parsed_ec['toolchain']['name'] == GCCcore.NAME + and dep['name'] == 'binutils' + and dep['toolchain']['name'] == GCCcore.NAME): + dep['version'] = toolchain_mapping['binutils']['version'] + dep['versionsuffix'] = toolchain_mapping['binutils']['versionsuffix'] + + # Determine the name of the modified easyconfig and dump it target_dir + ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) + if targetdir is None: + targetdir = tempfile.gettempdir() + tweaked_spec = os.path.join(target_dir, ec_filename) + parsed_ec.dump(tweaked_spec) + _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) \ No newline at end of file diff --git a/test/framework/tweak.py b/test/framework/tweak.py index e6adbadb4e..32a6ef1c57 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -36,7 +36,7 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs -from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies +from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies, map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes from easybuild.tools.filetools import write_file @@ -198,7 +198,7 @@ def test_match_minimum_tc_specs(self): goolf_hierarchy[2], iimpi_hierarchy) def test_map_toolchain_hierarchies(self): - """Test between two toolchain hierarchies""" + """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), @@ -228,6 +228,18 @@ def test_map_toolchain_hierarchies(self): 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, 'binutils': {'version': '2.25', 'versionsuffix': ''}}) + def test_map_easyconfig_to_target_tc_hierarchy(self): + """Test mapping of easyconfig to target hierarchy""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + }) + get_toolchain_hierarchy.clear() + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.25'} + iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + map_easyconfig_to_target_tc_hierarchy() + def suite(): """ return all the tests in this file """ return TestLoaderFiltered().loadTestsFromTestCase(TweakTest, sys.argv[1:]) From 9b204912bc7b8ac8d2d155d2303c19630a3e31fb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 20:24:54 +0200 Subject: [PATCH 16/51] Add tests for mapping an easyconfig to new hierarchy --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/tweak.py | 51 +++++++++++-------- .../b/binutils/binutils-2.26-GCCcore-4.9.3.eb | 26 ++++++++++ .../test_ecs/g/GCC/GCC-4.9.3-2.26.eb | 27 ++++++++++ .../h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb | 21 ++++++++ test/framework/filetools.py | 2 +- test/framework/robot.py | 2 +- test/framework/tweak.py | 19 +++++-- 8 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb create mode 100644 test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb create mode 100644 test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 682d699280..1d951d3b04 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -153,7 +153,7 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None # the parent toolchain is at the top of the hierarchy - toolchain_hierarchy = [parent_toolchain] + toolchain_hierarchy = [dict(parent_toolchain)] while subtoolchain_name: # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 5f6c94fb6a..57313d67d5 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -44,7 +44,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -137,7 +137,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path - modifying_toolchain = False # TODO Remove this to enable all tests if orig_ec['spec'] in listed_ec_paths: if modifying_toolchain: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, @@ -767,25 +766,35 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] # Replace the toolchain - parsed_ec['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] + parsed_ec['ec']['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] # Replace the toolchains of all the dependencies - for dep in parsed_ec.dependencies(): - # skip dependencies that are marked as external modules - if dep['external_module']: - continue - if dep['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: - dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] - # Replace the binutils version (if necessary) - if (parsed_ec['toolchain']['name'] == GCCcore.NAME - and dep['name'] == 'binutils' - and dep['toolchain']['name'] == GCCcore.NAME): - dep['version'] = toolchain_mapping['binutils']['version'] - dep['versionsuffix'] = toolchain_mapping['binutils']['versionsuffix'] - + filter_deps = build_option('filter_deps') + for key in ['builddependencies', 'dependencies', 'hiddendependencies']: + # loop over a *copy* of dependency dicts (with resolved templates); + # to update the original dep dict, we need to index with idx into self._config[key][0]... + for idx, dep in enumerate(parsed_ec['ec'][key]): + # reference to original dep dict, this is the one we should be updating + orig_dep = parsed_ec['ec']._config[key][0][idx] + # skip dependencies that are marked as external modules + if dep['external_module']: + continue + if dep['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + orig_dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] + # Replace the binutils version (if necessary) + if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and + dep['toolchain']['name'] == GCCcore.NAME): + orig_dep['version'] = toolchain_mapping['binutils']['version'] + orig_dep['versionsuffix'] = toolchain_mapping['binutils']['versionsuffix'] + if not dep['external_module']: + # set module names + orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) + orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it target_dir - ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) - if targetdir is None: - targetdir = tempfile.gettempdir() + ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) + if target_dir is None: + target_dir = tempfile.gettempdir() tweaked_spec = os.path.join(target_dir, ec_filename) - parsed_ec.dump(tweaked_spec) - _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) \ No newline at end of file + parsed_ec['ec'].dump(tweaked_spec) + _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) + + return tweaked_spec \ No newline at end of file diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb new file mode 100644 index 0000000000..e6e318b505 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb @@ -0,0 +1,26 @@ +# should be software specific, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'binutils' +version = '2.26' + +homepage = 'http://directory.fsf.org/project/binutils/' +description = "binutils: GNU binary utilities" + +toolchain = {'name': 'GCCcore', 'version': '4.9.3'} + +sources = [SOURCE_TAR_GZ] +source_urls = [GNU_SOURCE] + +# Testing purposes only so remove deps +#builddependencies = [ +# ('M4', '1.4.17'), +# ('flex', '2.5.39'), +# ('Bison', '3.0.4'), +# # zlib required, but being linked in statically, so not a runtime dep +# ('zlib', '1.2.8'), +# # use same binutils version that was used when building GCC toolchain, to 'bootstrap' this binutils +# ('binutils', version, '', True) +#] + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb new file mode 100644 index 0000000000..4ae3f5c7e9 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb @@ -0,0 +1,27 @@ +# should actually be 'Bundle', but close enough for testing purposes +easyblock = 'Toolchain' + +name = 'GCC' +version = '4.9.3' + +binutilsver = '2.26' +versionsuffix = '-%s' % binutilsver + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': ''} + +dependencies = [ + ('GCCcore', version), + # binutils built on top of GCCcore, which was built on top of (dummy-built) binutils + # excluded, good enough for testing purposes + ('binutils', binutilsver, '', ('GCCcore', version)), +] + +altroot = 'GCCcore' +altversion = 'GCCcore' + +# this bundle serves as a compiler-only toolchain, so it should be marked as compiler (important for HMNS) +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb new file mode 100644 index 0000000000..c445c7d4cb --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb @@ -0,0 +1,21 @@ +easyblock = 'ConfigureMake' + +name = 'hwloc' +version = '1.6.2' + +homepage = 'http://www.open-mpi.org/projects/hwloc/' +description = """The Portable Hardware Locality (hwloc) software package provides a portable abstraction + (across OS, versions, architectures, ...) of the hierarchical topology of modern architectures, including + NUMA memory nodes, sockets, shared caches, cores and simultaneous multithreading. It also gathers various + system attributes such as cache and memory information as well as the locality of I/O devices such as + network interfaces, InfiniBand HCAs or GPUs. It primarily aims at helping applications with gathering + information about modern computing hardware so as to exploit it accordingly and efficiently.""" + +toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} + +source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/'] +sources = [SOURCE_TAR_GZ] + +builddependencies = [('binutils', '2.26')] + +moduleclass = 'system' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e25d16cb1e..f80eb77e9e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1221,7 +1221,7 @@ def test_copy_dir(self): ft.copy_dir(to_copy, target_dir, ignore=lambda src, names: [x for x in names if '4.7.2' in x]) self.assertTrue(os.path.exists(target_dir)) - expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.25.eb'] + expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.26.eb'] self.assertEqual(sorted(os.listdir(target_dir)), expected) # GCC-4.7.2.eb should not get copied, since it's specified as file too ignore self.assertFalse(os.path.exists(os.path.join(target_dir, 'GCC-4.7.2.eb'))) diff --git a/test/framework/robot.py b/test/framework/robot.py index bff370c059..e75eb92e40 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1116,7 +1116,7 @@ def test_check_conflicts(self): # direct conflict on software version ecs, _ = parse_easyconfigs([ (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.7.2.eb'), False), - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), + (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.26.eb'), False), ]) self.mock_stderr(True) conflicts = check_conflicts(ecs, self.modtool) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 32a6ef1c57..a940f92f76 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -32,7 +32,7 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner -from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs @@ -60,7 +60,7 @@ def test_find_matching_easyconfigs(self): self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver))) ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path]) - gccvers = ['4.6.3', '4.6.4', '4.7.2', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25'] + gccvers = ['4.6.3', '4.6.4', '4.7.2', '4.8.2', '4.8.3', '4.9.2', '4.9.3-2.25', '4.9.3-2.26'] self.assertEqual(len(ecs), len(gccvers)) ecs_basename = [os.path.basename(ec) for ec in ecs] for gccver in gccvers: @@ -221,7 +221,7 @@ def test_map_toolchain_hierarchies(self): goolf_tc, iimpi_tc, self.modtool) # Test that we correctly include GCCcore binutils when it is there - gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.25'} + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), {'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, @@ -236,9 +236,18 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() - gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.25'} + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} - map_easyconfig_to_target_tc_hierarchy() + tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) + ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) + tweaked_ec = process_easyconfig(tweaked_spec)[0] + tweaked_dict = tweaked_ec['ec'].asdict() + key, value = 'toolchain', iccifort_binutils_tc + self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + key, value = 'version', '2.25' + self.assertTrue(key in tweaked_dict['builddependencies'][0] and + value == tweaked_dict['builddependencies'][0][key]) def suite(): """ return all the tests in this file """ From ee1eb882a4c1abe277a5d4b336cffe15b433029f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 20:39:33 +0200 Subject: [PATCH 17/51] Miscellaneous bug fixes --- easybuild/framework/easyconfig/tweak.py | 55 +++++++++++++------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 57313d67d5..0853883fcf 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -85,7 +85,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): toolchains) # Toolchain is unique, let's store it source_toolchain = easyconfigs[-1]['ec']['toolchain'] - modifying_toolchain = False + modifying_toolchains = False target_toolchain = {} src_to_dst_tc_mapping = {} @@ -98,7 +98,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # We're doing something with the toolchain, build specifications should be applied to the whole dependency graph # Obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) - modifying_toolchain = True + modifying_toolchains = True keys = build_specs.keys() if 'toolchain_name' in keys: target_toolchain['name'] = build_specs['toolchain_name'] @@ -120,7 +120,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): i = 0 while i < len(orig_ecs): - if orig_ecs[i]['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: + if orig_ecs[i]['ec']['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: # drop elements in toolchain hierarchy del orig_ecs[i] else: @@ -132,26 +132,27 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: - # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the - # prepended path so that they are found first). - # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot - # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path - - if orig_ec['spec'] in listed_ec_paths: - if modifying_toolchain: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_path) - else: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) - new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) - tweaked_easyconfigs.extend(new_ecs) - else: - # Place all tweaked dependency easyconfigs in the directory appended to the robot path - if modifying_toolchain: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - targetdir=tweaked_ecs_deps_path) + if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the + # prepended path so that they are found first). + # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot + # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path + + if orig_ec['spec'] in listed_ec_paths: + if modifying_toolchains: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + tweaked_ecs_path) + else: + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) + new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) + tweaked_easyconfigs.extend(new_ecs) else: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) + # Place all tweaked dependency easyconfigs in the directory appended to the robot path + if modifying_toolchains: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + targetdir=tweaked_ecs_deps_path) + else: + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) return tweaked_easyconfigs @@ -754,13 +755,13 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): return tc_mapping -def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir=None): +def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out :param ec_spec: Location of original easyconfig :param toolchain_mapping: Mapping between toolchain and target toolchain and target toolchain - :param target_dir: + :param targetdir: :return mapped_spec: """ # Fully parse the original easyconfig @@ -791,9 +792,9 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, target_dir orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) - if target_dir is None: - target_dir = tempfile.gettempdir() - tweaked_spec = os.path.join(target_dir, ec_filename) + if targetdir is None: + targetdir = tempfile.gettempdir() + tweaked_spec = os.path.join(targetdir, ec_filename) parsed_ec['ec'].dump(tweaked_spec) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) From 1d07e4ac308977edfea76f5db4b5cf8a4fe79cbe Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 20:41:55 +0200 Subject: [PATCH 18/51] Style fixes --- easybuild/framework/easyconfig/tweak.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 0853883fcf..a6716fd572 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -133,8 +133,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): tweaked_easyconfigs = [] for orig_ec in orig_ecs: if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: - # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the - # prepended path so that they are found first). + # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use + # the prepended path so that they are found first). # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path @@ -798,4 +798,4 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= parsed_ec['ec'].dump(tweaked_spec) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) - return tweaked_spec \ No newline at end of file + return tweaked_spec From 9abfe53bebce45cb5834f21610e2bf4d6b1a740a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 20:49:14 +0200 Subject: [PATCH 19/51] Move careless dummy check placement --- easybuild/framework/easyconfig/tweak.py | 39 ++++++++++++------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a6716fd572..2fd716751c 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -132,27 +132,26 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: - if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: - # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use - # the prepended path so that they are found first). - # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot - # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path - - if orig_ec['spec'] in listed_ec_paths: - if modifying_toolchains: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_path) - else: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) - new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) - tweaked_easyconfigs.extend(new_ecs) + # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use + # the prepended path so that they are found first). + # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot + # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path + + if orig_ec['spec'] in listed_ec_paths: + if modifying_toolchains and orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + tweaked_ecs_path) else: - # Place all tweaked dependency easyconfigs in the directory appended to the robot path - if modifying_toolchains: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - targetdir=tweaked_ecs_deps_path) - else: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) + new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) + tweaked_easyconfigs.extend(new_ecs) + else: + # Place all tweaked dependency easyconfigs in the directory appended to the robot path + if modifying_toolchains and orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + targetdir=tweaked_ecs_deps_path) + else: + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) return tweaked_easyconfigs From 97e7d8231e31d9314ccfb98d5b023ffa87add595 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Jul 2018 20:53:47 +0200 Subject: [PATCH 20/51] Move careless dummy check placement --- easybuild/framework/easyconfig/tweak.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 2fd716751c..edc02d8c7e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -138,18 +138,20 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path if orig_ec['spec'] in listed_ec_paths: - if modifying_toolchains and orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_path) + if modifying_toolchains: + if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + tweaked_ecs_path) else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) tweaked_easyconfigs.extend(new_ecs) else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path - if modifying_toolchains and orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - targetdir=tweaked_ecs_deps_path) + if modifying_toolchains: + if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + targetdir=tweaked_ecs_deps_path) else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) From 9d25494e1be3837ada7f9cbba39c9fb971ca5696 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 09:47:41 +0200 Subject: [PATCH 21/51] Make sure to add capabilities also to the parent toolchain --- easybuild/framework/easyconfig/easyconfig.py | 53 ++++++++++---------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 1d951d3b04..4af933cc96 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -225,7 +225,8 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions))) - if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'): + if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and (not build_option('add_dummy_to_minimal_toolchains') + or require_capabilities): # we're done break @@ -234,31 +235,31 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version}) - # also add toolchain capabilities - if require_capabilities: - for toolchain in toolchain_hierarchy: - toolchain_class, _ = search_toolchain(toolchain['name']) - tc = toolchain_class(version=toolchain['version']) - try: - toolchain['compiler_family'] = tc.comp_family() - except EasyBuildError: - toolchain['compiler_family'] = None - try: - toolchain['mpi_family'] = tc.mpi_family() - except EasyBuildError: - toolchain['mpi_family'] = None - try: - toolchain['blas_family'] = tc.blas_family() - except EasyBuildError: - toolchain['blas_family'] = None - try: - toolchain['lapack_family'] = tc.lapack_family() - except EasyBuildError: - toolchain['lapack_family'] = None - if 'CUDA_CC' in tc.variables: - toolchain['cuda'] = True - else: - toolchain['cuda'] = None # Useful to have it consistent with the rest + # also add toolchain capabilities + if require_capabilities: + for toolchain in toolchain_hierarchy: + toolchain_class, _ = search_toolchain(toolchain['name']) + tc = toolchain_class(version=toolchain['version']) + try: + toolchain['compiler_family'] = tc.comp_family() + except EasyBuildError: + toolchain['compiler_family'] = None + try: + toolchain['mpi_family'] = tc.mpi_family() + except EasyBuildError: + toolchain['mpi_family'] = None + try: + toolchain['blas_family'] = tc.blas_family() + except EasyBuildError: + toolchain['blas_family'] = None + try: + toolchain['lapack_family'] = tc.lapack_family() + except EasyBuildError: + toolchain['lapack_family'] = None + if 'CUDA_CC' in tc.variables: + toolchain['cuda'] = True + else: + toolchain['cuda'] = None # Useful to have it consistent with the rest _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy From 46a7f7fb8bdedd90a78ca4bd89e3472578806f65 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 11:12:48 +0200 Subject: [PATCH 22/51] Allow creation of tweaked toolchain easyconfig whenever there is an allowed mapping --- easybuild/framework/easyconfig/tweak.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index edc02d8c7e..6475d9cd44 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -137,19 +137,34 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path + new_ec_file = '' + verification_build_specs = dict(build_specs) if orig_ec['spec'] in listed_ec_paths: if modifying_toolchains: - if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + if src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, tweaked_ecs_path) + # Need to update the toolchain in the build_specs to match the toolchain mapping + keys = verification_build_specs.keys() + if 'toolchain_name' in keys: + verification_build_specs['toolchain_name'] = \ + src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]['name'] + if 'toolchain_version' in keys: + verification_build_specs['toolchain_version'] = \ + src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]['version'] + if 'toolchain' in keys: + verification_build_specs['toolchain'] = \ + src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']] else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) - new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) - tweaked_easyconfigs.extend(new_ecs) + if new_ec_file: + # Need to update the toolchain the build_specs to match the toolchain mapping + new_ecs = process_easyconfig(new_ec_file, build_specs=verification_build_specs) + tweaked_easyconfigs.extend(new_ecs) else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path if modifying_toolchains: - if orig_ec['ec']['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + if src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, targetdir=tweaked_ecs_deps_path) else: From eddc050beca0cda5fad957f7778e48eb1fe5a55b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 12:42:52 +0200 Subject: [PATCH 23/51] Always map deps if a map exists --- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/robot.py | 2 -- test/framework/options.py | 14 +++++--------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6475d9cd44..23c71599b1 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -795,7 +795,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # skip dependencies that are marked as external modules if dep['external_module']: continue - if dep['toolchain']['name'] != DUMMY_TOOLCHAIN_NAME: + if toolchain_mapping[dep['toolchain']['name']]: orig_dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index f797aa0a6e..3e62ae17a0 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -318,8 +318,6 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) - # build specs should not be passed down to resolved dependencies, - # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) diff --git a/test/framework/options.py b/test/framework/options.py index e51ad805ef..077875ab4f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1481,24 +1481,20 @@ def test_recursive_try(self): ] for extra_args in [[], ['--module-naming-scheme=HierarchicalMNS']]: - outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + # toolchain GCC/4.7.2 (subtoochain of gompi/1.4.10) should be listed (and present) + mark = 'x' - # toolchain gompi/1.4.10 should be listed (but not present yet) - if extra_args: - mark = 'x' - else: - mark = ' ' - tc_regex = re.compile("^ \* \[%s\] .*/gompi-1.4.10.eb \(module: .*gompi/1.4.10\)$" % mark, re.M) + tc_regex = re.compile("^ \* \[%s\] .*/GCC-4.7.2.eb \(module: .*GCC/4.7.2\)$" % mark, re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) # both toy and gzip dependency should be listed with gompi/1.4.10 toolchain for ec_name in ['gzip-1.4', 'toy-0.0']: - ec = '%s-gompi-1.4.10.eb' % ec_name + ec = '%s-GCC-4.7.2.eb' % ec_name if extra_args: mod = ec_name.replace('-', '/') else: - mod = '%s-gompi-1.4.10' % ec_name.replace('-', '/') + mod = '%s-GCC-4.7.2' % ec_name.replace('-', '/') mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) #mod_regex = re.compile("%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) From e68523179f55337f9be11937f6dccb22d72b464d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 13:31:46 +0200 Subject: [PATCH 24/51] Always map deps if a map exists --- .../test_ecs/i/ictce/ictce-4.1.13.eb | 2 +- .../test_ecs/t/toy/toy-0.0-gompi-1.3.12.eb | 33 +++++++++++++++++++ test/framework/toy_build.py | 17 ++++++---- 3 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12.eb diff --git a/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb b/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb index d2f022d78d..8d54862a1a 100644 --- a/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb +++ b/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb @@ -14,7 +14,7 @@ compver = '2011.13.367' fake_dependencies = [ ('icc', compver), ('ifort', compver), - ('impi', '4.1.0.027'), + ('impi', '4.1.3.049'), ('imkl', '10.3.12.361') ] diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12.eb new file mode 100644 index 0000000000..866bc3ece4 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12.eb @@ -0,0 +1,33 @@ +name = 'toy' +version = '0.0' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = {'name': 'gompi', 'version': '1.3.12'} +toolchainopts = {'pic': True, 'opt': True, 'optarch': True} + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6250c17943..23ec494ddc 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -843,7 +843,7 @@ def test_toy_hierarchical_subdir_user_modules(self): write_file(openmpi_mod, extra_modtxt, append=True) args = [ - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'), + os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-1.3.12.eb'), '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % home, @@ -861,9 +861,10 @@ def test_toy_hierarchical_subdir_user_modules(self): toy_mod = os.path.join(home, 'modules', 'all', openmpi_mod_subdir, 'toy', '0.0' + mod_ext) toy_modtxt = read_file(toy_mod) - for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - regex = re.compile('load.*' + modname, re.M) - self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + #No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary + #for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + # regex = re.compile('load.*' + modname, re.M) + # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) @@ -910,9 +911,11 @@ def test_toy_hierarchical_subdir_user_modules(self): self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) toy_modtxt = read_file(toy_mod) - for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - regex = re.compile('load.*' + modname, re.M) - self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + #No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary + + #for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + # regex = re.compile('load.*' + modname, re.M) + # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) From f003b1710f0159eb79c6ea5224c959c5ed89015a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 13:34:13 +0200 Subject: [PATCH 25/51] Fix spacing of comments --- test/framework/toy_build.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 23ec494ddc..1a89b25059 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -861,10 +861,10 @@ def test_toy_hierarchical_subdir_user_modules(self): toy_mod = os.path.join(home, 'modules', 'all', openmpi_mod_subdir, 'toy', '0.0' + mod_ext) toy_modtxt = read_file(toy_mod) - #No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary - #for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - # regex = re.compile('load.*' + modname, re.M) - # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + # No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary + # for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + # regex = re.compile('load.*' + modname, re.M) + # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) @@ -911,11 +911,11 @@ def test_toy_hierarchical_subdir_user_modules(self): self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) toy_modtxt = read_file(toy_mod) - #No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary + # No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary - #for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - # regex = re.compile('load.*' + modname, re.M) - # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + # for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + # regex = re.compile('load.*' + modname, re.M) + # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) From a40c2d6717feecd1f5b89c153d213309fa0ca2fa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 15:20:53 +0200 Subject: [PATCH 26/51] Fix spacing of comments --- easybuild/framework/easyconfig/tweak.py | 19 +++++++++++++------ test/framework/options.py | 17 ++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 23c71599b1..2535ea3e66 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -89,17 +89,19 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain = {} src_to_dst_tc_mapping = {} - if 'name' in build_specs or 'version' in build_specs: - # no recursion if software name/version build specification are included - # in that case, do not construct full dependency graph - orig_ecs = easyconfigs - _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) - else: + if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs: # We're doing something with the toolchain, build specifications should be applied to the whole dependency graph # Obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) modifying_toolchains = True keys = build_specs.keys() + + # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not + # supported + for key in keys: + if key not in ['toolchain_name', 'toolchain_version', 'toolchain']: + raise EasyBuildError("Combining --try-toolchain* with other build options is currently not supported") + if 'toolchain_name' in keys: target_toolchain['name'] = build_specs['toolchain_name'] else: @@ -125,6 +127,11 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): del orig_ecs[i] else: i += 1 + else: + # no recursion if software name/version build specification are included or we are amending something + # in that case, do not construct full dependency graph + orig_ecs = easyconfigs + _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) # keep track of originally listed easyconfigs (via their path) listed_ec_paths = [ec['spec'] for ec in easyconfigs] diff --git a/test/framework/options.py b/test/framework/options.py index 077875ab4f..478ffbf36b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1421,13 +1421,15 @@ def test_try(self): test_cases = [ ([], 'toy/0.0'), - (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), - (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-gompi-1.4.10'), + # --try-toolchain* has a different branch to all other try options, don't combine them + # (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), # --try-toolchain is overridden by --toolchain (['--try-toolchain=gompi,1.3.12', '--toolchain=dummy,dummy'], 'toy/0.0'), (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), - (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-gompi-1.4.10'), - (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), + (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), + # --try-toolchain* has a different branch to all other try options, don't combine them + #(['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), # --try-amend is overridden by --amend (['--amend=versionsuffix=', '--try-amend=versionsuffix=-test'], 'toy/0.0'), @@ -1441,8 +1443,9 @@ def test_try(self): # define extra list-typed parameter (['--try-amend=versionsuffix=-test5', '--try-amend=exts_list=1,2,3'], 'toy/0.0-test5'), # only --try causes other build specs to be included too - (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), - (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + # --try-toolchain* has a different branch to all other try options, don't combine them + # (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + # (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), (['--software=foo,1.2.3', '--try-amend=versionsuffix=-test'], 'foo/1.2.3-test'), ] @@ -1526,7 +1529,7 @@ def test_cleanup_builddir(self): self.eb_main(args, do_build=True, verbose=True) # make sure build directory is properly cleaned up after a successful build (default behavior) - self.assertFalse(os.path.exists(toy_buildpath), "Build dir %s removed after succesful build" % toy_buildpath) + self.assertFalse(os.path.exists(toy_buildpath), "Build dir %s removed after successful build" % toy_buildpath) # make sure --disable-cleanup-builddir works args.append('--disable-cleanup-builddir') self.eb_main(args, do_build=True, verbose=True) From cc2168cdda603996f3a617733ebfa1034d7d23fc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 15:53:59 +0200 Subject: [PATCH 27/51] Fix more tests --- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 15 ++++++++++----- test/framework/robot.py | 2 +- test/framework/scripts.py | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 147616795d..2ad3a5216c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1720,7 +1720,7 @@ def test_find_related_easyconfigs(self): ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} ec['versionsuffix'] = '-foobar' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - self.assertEqual(res, ['toy-0.0-gompi-1.3.12-test.eb']) + self.assertEqual(res, ['toy-0.0-gompi-1.3.12-test.eb', 'toy-0.0-gompi-1.3.12.eb']) # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) ec['versionsuffix'] = '-deps' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index f80eb77e9e..4fb162c470 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1221,7 +1221,8 @@ def test_copy_dir(self): ft.copy_dir(to_copy, target_dir, ignore=lambda src, names: [x for x in names if '4.7.2' in x]) self.assertTrue(os.path.exists(target_dir)) - expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.26.eb'] + expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.25.eb', + 'GCC-4.9.3-2.26.eb'] self.assertEqual(sorted(os.listdir(target_dir)), expected) # GCC-4.7.2.eb should not get copied, since it's specified as file too ignore self.assertFalse(os.path.exists(os.path.join(target_dir, 'GCC-4.7.2.eb'))) @@ -1410,7 +1411,7 @@ def test_search_file(self): # check for default semantics, test case-insensitivity var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True) self.assertEqual(var_defs, []) - self.assertEqual(len(hits), 2) + self.assertEqual(len(hits), 3) self.assertTrue(all(os.path.exists(p) for p in hits)) self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.6.4.eb')) self.assertTrue(hits[1].endswith('/hwloc-1.6.2-GCC-4.7.2.eb')) @@ -1418,7 +1419,8 @@ def test_search_file(self): # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.6.4.eb', 'hwloc-1.6.2-GCC-4.7.2.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.6.4.eb', 'hwloc-1.6.2-GCC-4.7.2.eb', + 'hwloc-1.6.2-GCC-4.9.3-2.26.eb']) # check specifying of ignored dirs var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) @@ -1427,7 +1429,8 @@ def test_search_file(self): # check short mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) - self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.6.4.eb', '$CFGS1/hwloc-1.6.2-GCC-4.7.2.eb']) + self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.6.4.eb', '$CFGS1/hwloc-1.6.2-GCC-4.7.2.eb', + '$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb']) # check terse mode (implies 'silent', overrides 'short') var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) @@ -1435,13 +1438,15 @@ def test_search_file(self): expected = [ os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.7.2.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.6.4.eb', 'hwloc-1.6.2-GCC-4.7.2.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.6.4.eb', 'hwloc-1.6.2-GCC-4.7.2.eb', + 'hwloc-1.6.2-GCC-4.9.3-2.26.eb']) def test_find_eb_script(self): """Test find_eb_script function.""" diff --git a/test/framework/robot.py b/test/framework/robot.py index e75eb92e40..bff370c059 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1116,7 +1116,7 @@ def test_check_conflicts(self): # direct conflict on software version ecs, _ = parse_easyconfigs([ (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.7.2.eb'), False), - (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.26.eb'), False), + (os.path.join(test_easyconfigs, 'g', 'GCC', 'GCC-4.9.3-2.25.eb'), False), ]) self.mock_stderr(True) conflicts = check_conflicts(ecs, self.modtool) diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 4a54cf91a0..10b0c7a05d 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -85,7 +85,7 @@ def test_generate_software_list(self): out, ec = run_cmd(cmd, simple=False) # make sure output is kind of what we expect it to be - regex = r"Supported Packages \(26 " + regex = r"Supported Packages \(27 " self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { 'B': '1', # bzip2 From 53646aabfd8cdce45809f17f54717c5c5fe945f7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 15:58:15 +0200 Subject: [PATCH 28/51] Fix style --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 478ffbf36b..7ed3520a57 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1429,7 +1429,7 @@ def test_try(self): (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), # --try-toolchain* has a different branch to all other try options, don't combine them - #(['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), + # (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), # --try-amend is overridden by --amend (['--amend=versionsuffix=', '--try-amend=versionsuffix=-test'], 'toy/0.0'), From b060dfa64cd483076d9144cd2235376713775c92 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 16:15:35 +0200 Subject: [PATCH 29/51] Fall back to regex when things are too complicated --- easybuild/framework/easyconfig/tweak.py | 59 ++++++++++++++----------- test/framework/options.py | 14 +++--- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 2535ea3e66..3f900127ba 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -88,46 +88,53 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): modifying_toolchains = False target_toolchain = {} src_to_dst_tc_mapping = {} + revert_to_regex = False if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs: - # We're doing something with the toolchain, build specifications should be applied to the whole dependency graph - # Obtain full dependency graph for specified easyconfigs - # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) - modifying_toolchains = True keys = build_specs.keys() # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported for key in keys: if key not in ['toolchain_name', 'toolchain_version', 'toolchain']: - raise EasyBuildError("Combining --try-toolchain* with other build options is currently not supported") + print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") + revert_to_regex = True - if 'toolchain_name' in keys: - target_toolchain['name'] = build_specs['toolchain_name'] - else: - target_toolchain['name'] = source_toolchain['name'] - if 'toolchain_version' in keys: - target_toolchain['version'] = build_specs['toolchain_version'] - else: - target_toolchain['version'] = source_toolchain['version'] + if not revert_to_regex: + # We're doing something with the toolchain, build specifications should be applied to whole dependency graph + # Obtain full dependency graph for specified easyconfigs + # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) + modifying_toolchains = True + + if 'toolchain_name' in keys: + target_toolchain['name'] = build_specs['toolchain_name'] + else: + target_toolchain['name'] = source_toolchain['name'] + if 'toolchain_version' in keys: + target_toolchain['version'] = build_specs['toolchain_version'] + else: + target_toolchain['version'] = source_toolchain['version'] - src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) + src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) - _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) - orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) + _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) + orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) - # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) - # We can leave any dependencies they may have as they will only be used if required (or originally listed) - _log.debug("Filtering out toolchain hierarchy for %s" % source_toolchain) + # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) + # We can leave any dependencies they may have as they will only be used if required (or originally listed) + _log.debug("Filtering out toolchain hierarchy for %s" % source_toolchain) - i = 0 - while i < len(orig_ecs): - if orig_ecs[i]['ec']['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: - # drop elements in toolchain hierarchy - del orig_ecs[i] - else: - i += 1 + i = 0 + while i < len(orig_ecs): + if orig_ecs[i]['ec']['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: + # drop elements in toolchain hierarchy + del orig_ecs[i] + else: + i += 1 else: + revert_to_regex = True + + if revert_to_regex: # no recursion if software name/version build specification are included or we are amending something # in that case, do not construct full dependency graph orig_ecs = easyconfigs diff --git a/test/framework/options.py b/test/framework/options.py index 7ed3520a57..f7eedbe338 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1421,15 +1421,15 @@ def test_try(self): test_cases = [ ([], 'toy/0.0'), - # --try-toolchain* has a different branch to all other try options, don't combine them - # (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + # --try-toolchain* has a different branch to all other try options, combining defaults back to regex + (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), # --try-toolchain is overridden by --toolchain (['--try-toolchain=gompi,1.3.12', '--toolchain=dummy,dummy'], 'toy/0.0'), (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), - # --try-toolchain* has a different branch to all other try options, don't combine them - # (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), + # --try-toolchain* has a different branch to all other try options, combining defaults back to regex + (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), # --try-amend is overridden by --amend (['--amend=versionsuffix=', '--try-amend=versionsuffix=-test'], 'toy/0.0'), @@ -1443,9 +1443,9 @@ def test_try(self): # define extra list-typed parameter (['--try-amend=versionsuffix=-test5', '--try-amend=exts_list=1,2,3'], 'toy/0.0-test5'), # only --try causes other build specs to be included too - # --try-toolchain* has a different branch to all other try options, don't combine them - # (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), - # (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + # --try-toolchain* has a different branch to all other try options, combining defaults back to regex + (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), (['--software=foo,1.2.3', '--try-amend=versionsuffix=-test'], 'foo/1.2.3-test'), ] From 2073e384b764069ab56e150898d02a75c7104836 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 16:41:05 +0200 Subject: [PATCH 30/51] Fi more tests --- test/framework/toy_build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1a89b25059..8603e0a19b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -660,9 +660,11 @@ def test_toy_hierarchical(self): '--module-naming-scheme=HierarchicalMNS', ] - # test module paths/contents with gompi build + # test module paths/contents with goolf build extra_args = [ '--try-toolchain=goolf,1.4.10', + # Force using the regex method so try uses the full toolchain + '--try-amend=parallel=1', ] self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) @@ -707,7 +709,6 @@ def test_toy_hierarchical(self): modtxt = read_file(toy_module_path) self.assertFalse(re.search("module load", modtxt)) os.remove(toy_module_path) - # test module path with GCC/4.7.2 build, pretend to be an MPI lib by setting moduleclass extra_args = [ '--try-toolchain=GCC,4.7.2', From b46b093cc05eb0c794c853872e2d2ee52ac6665e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 16:45:50 +0200 Subject: [PATCH 31/51] Fix more tests --- test/framework/scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 10b0c7a05d..601f3a30c1 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -88,7 +88,7 @@ def test_generate_software_list(self): regex = r"Supported Packages \(27 " self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { - 'B': '1', # bzip2 + 'B': '2', # binutils, bzip2 'C': '2', # CrayCCE, CUDA 'F': '1', # FFTW 'G': '6', # GCC, GCCcore, gmvapich2, gompi, goolf, gzip From 746471d15430d06e6ffefb3c247fd8650119e204 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 16:56:35 +0200 Subject: [PATCH 32/51] Fix last test --- test/framework/docs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 390c3addaf..766fe09c57 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -252,7 +252,7 @@ def test_list_software(self): '', 'homepage: https://easybuilders.github.io/easybuild', '', - " * toy v0.0: dummy", + " * toy v0.0: dummy, gompi/1.3.12", " * toy v0.0 (versionsuffix: '-deps'): dummy", " * toy v0.0 (versionsuffix: '-iter'): dummy", " * toy v0.0 (versionsuffix: '-multiple'): dummy", @@ -271,17 +271,18 @@ def test_list_software(self): '', '*homepage*: https://easybuilders.github.io/easybuild', '', - '======= ============= ================', - 'version versionsuffix toolchain ', - '======= ============= ================', - '``0.0`` ``dummy`` ', - '``0.0`` ``-deps`` ``dummy`` ', - '``0.0`` ``-iter`` ``dummy`` ', - '``0.0`` ``-multiple`` ``dummy`` ', - '``0.0`` ``-test`` ``gompi/1.3.12``', - '======= ============= ================', + '======= ============= ===========================', + 'version versionsuffix toolchain ', + '======= ============= ===========================', + '``0.0`` ``dummy``, ``gompi/1.3.12``', + '``0.0`` ``-deps`` ``dummy`` ', + '``0.0`` ``-iter`` ``dummy`` ', + '``0.0`` ``-multiple`` ``dummy`` ', + '``0.0`` ``-test`` ``gompi/1.3.12`` ', + '======= ============= ===========================', ] txt = list_software(output_format='rst', detailed=True) + print txt lines = txt.split('\n') expected_found = any([lines[i:i+len(expected)] == expected for i in range(len(lines))]) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) From 6c26f7957e9f584da96597e3f402f910f727525c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 16:58:43 +0200 Subject: [PATCH 33/51] Remove stray print --- test/framework/docs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 766fe09c57..cc5a3c094e 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -282,7 +282,6 @@ def test_list_software(self): '======= ============= ===========================', ] txt = list_software(output_format='rst', detailed=True) - print txt lines = txt.split('\n') expected_found = any([lines[i:i+len(expected)] == expected for i in range(len(lines))]) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) From 133b4c73e0b87aca3e3a6e5468cb16d308163bd0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 17:36:34 +0200 Subject: [PATCH 34/51] Fix checking for key in dictionary --- easybuild/framework/easyconfig/tweak.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3f900127ba..9d3766cc6d 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -155,7 +155,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): verification_build_specs = dict(build_specs) if orig_ec['spec'] in listed_ec_paths: if modifying_toolchains: - if src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]: + if orig_ec['ec']['toolchain']['name'] in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, tweaked_ecs_path) # Need to update the toolchain in the build_specs to match the toolchain mapping @@ -178,7 +178,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path if modifying_toolchains: - if src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]: + if orig_ec['ec']['toolchain']['name'] in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, targetdir=tweaked_ecs_deps_path) else: @@ -809,7 +809,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # skip dependencies that are marked as external modules if dep['external_module']: continue - if toolchain_mapping[dep['toolchain']['name']]: + if dep['toolchain']['name'] in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and From bb3bcde6adb9842d0f41ba40e5500921e61a775a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 31 Jul 2018 17:40:06 +0200 Subject: [PATCH 35/51] Be more careful when using toolchain_mapping --- easybuild/framework/easyconfig/tweak.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 9d3766cc6d..9766967084 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -796,8 +796,9 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] - # Replace the toolchain - parsed_ec['ec']['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] + # Replace the toolchain if the mapping exists + if parsed_ec['ec']['toolchain']['name'] in toolchain_mapping: + parsed_ec['ec']['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] # Replace the toolchains of all the dependencies filter_deps = build_option('filter_deps') for key in ['builddependencies', 'dependencies', 'hiddendependencies']: @@ -820,7 +821,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # set module names orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) - # Determine the name of the modified easyconfig and dump it target_dir + # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) if targetdir is None: targetdir = tempfile.gettempdir() From dd0a7ddf7add2521155ead49d744489510cc34c5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Sep 2018 16:37:38 +0200 Subject: [PATCH 36/51] Don't delete spaces --- easybuild/framework/easyconfig/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 68caf311b8..8e5f2d99e2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -172,9 +172,12 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. + The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains build option is enabled. + The most complex hierarchy we have now is goolfc which works as follows: + goolfc / \ gompic golfc(*) @@ -186,6 +189,7 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): GCCcore(*) | \ | (dummy: only considered if --add-dummy-to-minimal-toolchains configuration option is enabled) + :param parent_toolchain: dictionary with name/version of parent toolchain """ # obtain list of all possible subtoolchains From ae4d50e361c1a32849fd632e8f7836015a8a96d4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Sep 2018 17:20:52 +0200 Subject: [PATCH 37/51] Fix some tests --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- test/framework/tweak.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8e5f2d99e2..f95490264f 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -199,10 +199,10 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) # the parent toolchain is at the top of the hierarchy - toolchain_hierarchy = [parent_toolchain] + toolchain_hierarchy = [dict(parent_toolchain)] # use a queue to handle a breadth-first-search of the hierarchy, # which is required to take into account the potential for multiple subtoolchains - bfs_queue = [dict(parent_toolchain)] + bfs_queue = [parent_toolchain] visited = set() while bfs_queue: diff --git a/test/framework/tweak.py b/test/framework/tweak.py index a940f92f76..9dfec451e2 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -182,20 +182,19 @@ def test_match_minimum_tc_specs(self): goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, require_capabilities=True) - # Compiler first self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], goolf_hierarchy), {'name': 'GCC', 'version': '4.7.2'}) # Then MPI self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], goolf_hierarchy), {'name': 'gompi', 'version': '1.4.10'}) - # Check against itself for math - self.assertEqual(match_minimum_tc_specs(goolf_hierarchy[2], goolf_hierarchy), - {'name': 'goolf', 'version': '1.4.10'}) + # Check against own math only subtoolchain for math + self.assertEqual(match_minimum_tc_specs(goolf_hierarchy[1], goolf_hierarchy), + {'name': 'golf', 'version': '1.4.10'}) # Make sure there's an error when we can't do the mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs, - goolf_hierarchy[2], iimpi_hierarchy) + goolf_hierarchy[3], iimpi_hierarchy) def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" From 059a6212cfbe971dc4576963f0b2bfb19437fec9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Sep 2018 17:24:28 +0200 Subject: [PATCH 38/51] Fix another test --- test/framework/tweak.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 9dfec451e2..c41f9876dc 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -163,9 +163,9 @@ def test_compare_toolchain_specs(self): # Hierarchies are returned with top-level toolchain last, goolf has 3 elements here, intel has 2 # goolf <-> iimpi (should return False) - self.assertFalse(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1]), "goolf requires math libs") + self.assertFalse(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs") # gompi <-> iimpi - self.assertTrue(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1])) + self.assertTrue(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1])) # GCC <-> iimpi self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[1])) # GCC <-> iccifort From 1cf9d36fc355e93cca93f3d169111bb75f098bb7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Sep 2018 09:00:45 +0200 Subject: [PATCH 39/51] Fix more tests --- test/framework/filetools.py | 5 +++-- test/framework/robot.py | 2 ++ test/framework/scripts.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 6c76884503..8c34b17c8b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1411,11 +1411,12 @@ def test_search_file(self): # check for default semantics, test case-insensitivity var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True) self.assertEqual(var_defs, []) - self.assertEqual(len(hits), 3) + self.assertEqual(len(hits), 4) self.assertTrue(all(os.path.exists(p) for p in hits)) self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.6.4.eb')) self.assertTrue(hits[1].endswith('/hwloc-1.6.2-GCC-4.7.2.eb')) - self.assertTrue(hits[2].endswith('/hwloc-1.8-gcccuda-2.6.10.eb')) + self.assertTrue(hits[2].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) + self.assertTrue(hits[3].endswith('/hwloc-1.8-gcccuda-2.6.10.eb')) # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) diff --git a/test/framework/robot.py b/test/framework/robot.py index 0dcd5b49cf..a80ce85c23 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -724,6 +724,8 @@ def test_get_toolchain_hierarchy(self): self.assertEqual(goolf_hierarchy, [ {'name': 'GCC', 'version': '4.7.2', 'compiler_family': 'GCC', 'mpi_family': None, 'lapack_family': None, 'blas_family': None, 'cuda': None}, + {'name': 'golf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': None, + 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': None}, {'name': 'gompi', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', 'lapack_family': None, 'blas_family': None, 'cuda': None}, {'name': 'goolf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', diff --git a/test/framework/scripts.py b/test/framework/scripts.py index ae06199016..b073ad467b 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -85,7 +85,7 @@ def test_generate_software_list(self): out, ec = run_cmd(cmd, simple=False) # make sure output is kind of what we expect it to be - regex = r"Supported Packages \(31 " + regex = r"Supported Packages \(32 " self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { 'B': '2', # binutils, bzip2 From db1400a8f55a6cc0ea4f653243f42a780567de3d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Sep 2018 09:18:42 +0200 Subject: [PATCH 40/51] Fix final test --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 177a156561..6102d1a783 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -851,7 +851,7 @@ def test_try_robot_force(self): ("GCC-4.6.4.eb", "GCC/4.6.4", 'x'), ("OpenMPI-1.6.4-GCC-4.6.4.eb", "OpenMPI/1.6.4-GCC-4.6.4", 'x'), # OpenBLAS dependency is listed, but not there => ' ' - ("OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2", ' '), + ("OpenBLAS-0.2.6-GCC-4.6.4-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-GCC-4.6.4-LAPACK-3.4.2", ' '), # both FFTW and ScaLAPACK are listed => 'F' ("ScaLAPACK-%s.eb" % scalapack_ver, "ScaLAPACK/%s" % scalapack_ver, 'F'), ("FFTW-3.3.3-gompi-1.3.12.eb", "FFTW/3.3.3-gompi-1.3.12", 'F'), From 6c13972b6c0018f64ef4702b8827d8b06231ef4d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Sep 2018 17:16:54 +0200 Subject: [PATCH 41/51] Address some of the comments --- easybuild/framework/easyconfig/easyconfig.py | 37 ++++----- easybuild/framework/easyconfig/tweak.py | 28 ++++--- easybuild/tools/robot.py | 2 + .../test_ecs/g/GCC/GCC-4.9.3-2.25.eb | 1 - .../test_ecs/g/GCC/GCC-4.9.3-2.26.eb | 1 - .../test_ecs/i/ictce/ictce-4.1.13.eb | 2 +- test/framework/options.py | 11 +-- test/framework/robot.py | 76 +++++++++++++++---- test/framework/toy_build.py | 3 +- test/framework/tweak.py | 60 +++++++++++---- 10 files changed, 149 insertions(+), 72 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 10c66f0ad5..2f0a11312b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -82,6 +82,9 @@ # name of easyconfigs archive subdirectory EASYCONFIGS_ARCHIVE_DIR = '__archive__' +# Available capabilities of toolchains +CAPABILITIES = ['comp_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda'] + try: import autopep8 @@ -199,8 +202,9 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) - # the parent toolchain is at the top of the hierarchy - toolchain_hierarchy = [dict(parent_toolchain)] + # the parent toolchain is at the top of the hierarchy, we need a copy so that adding capabilities (below) doesn't + # affect the original object + toolchain_hierarchy = [copy.copy(parent_toolchain)] # use a queue to handle a breadth-first-search of the hierarchy, # which is required to take into account the potential for multiple subtoolchains bfs_queue = [parent_toolchain] @@ -282,26 +286,15 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): for toolchain in toolchain_hierarchy: toolchain_class, _ = search_toolchain(toolchain['name']) tc = toolchain_class(version=toolchain['version']) - try: - toolchain['compiler_family'] = tc.comp_family() - except EasyBuildError: - toolchain['compiler_family'] = None - try: - toolchain['mpi_family'] = tc.mpi_family() - except EasyBuildError: - toolchain['mpi_family'] = None - try: - toolchain['blas_family'] = tc.blas_family() - except EasyBuildError: - toolchain['blas_family'] = None - try: - toolchain['lapack_family'] = tc.lapack_family() - except EasyBuildError: - toolchain['lapack_family'] = None - if 'CUDA_CC' in tc.variables: - toolchain['cuda'] = True - else: - toolchain['cuda'] = None # Useful to have it consistent with the rest + for capability in CAPABILITIES: + # cuda is the special case which doesn't have a family attribute + if capability == 'cuda': + # use None rather than False, useful to have it consistent with the rest + toolchain['cuda'] = ('CUDA_CC' in tc.variables) or None + else: + if hasattr(tc, capability): + toolchain[capability] = getattr(tc, capability)() + _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 9766967084..82fa2e535b 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -44,7 +44,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS, CAPABILITIES from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -681,14 +681,16 @@ def obtain_ec_for(specs, paths, fp=None): def compare_toolchain_specs(source_tc_spec, target_tc_spec): """ - Compare whether a source and target toolchain have compatible characteristics + Compare whether a source toolchain is mappable to a target toolchain :param source_tc_spec: specs of source toolchain :param target_tc_spec: specs of target toolchain + + :return: boolean indicating whether or not source toolchain is compatible with target toolchain """ can_map = True # Check they have same capabilities - for key in ['compiler_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda']: + for key in CAPABILITIES: if target_tc_spec[key] is None and source_tc_spec[key] is not None: can_map = False break @@ -705,25 +707,26 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): """ minimal_matching_toolchain = {} target_compiler_family = '' - # Do a complete loop so we always end up with the minimal value in the hierarchy - # hierarchy is given from lowest to highest, so need to reverse the order in the list - for target_tc_spec in reversed(target_tc_hierarchy): + + # break out once we've found the first match since the hierarchy is ordered low to high in terms of capabilities + for target_tc_spec in target_tc_hierarchy: if compare_toolchain_specs(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used in the target if the original toolchain was also # GCCcore if target_tc_spec['name'] != GCCcore.NAME or \ (source_tc_spec['name'] == GCCcore.NAME and target_tc_spec['name'] == GCCcore.NAME): minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} - target_compiler_family = target_tc_spec['compiler_family'] + target_compiler_family = target_tc_spec['comp_family'] + break if not minimal_matching_toolchain: raise EasyBuildError("No possible mapping from source toolchain spec %s to target toolchain hierarchy specs %s", source_tc_spec, target_tc_hierarchy) # Warn if we are changing compiler families, this is very likely to cause problems - if target_compiler_family != source_tc_spec['compiler_family']: - print_warning("Your request will results in a compiler family switch (%s to %s). Here be dragons!" % - (source_tc_spec['compiler_family'], target_compiler_family)) + if target_compiler_family != source_tc_spec['comp_family']: + print_warning("Your request will result in a compiler family switch (%s to %s). Here be dragons!" % + (source_tc_spec['comp_family'], target_compiler_family)) return minimal_matching_toolchain @@ -733,6 +736,8 @@ def get_dep_tree_of_toolchain(toolchain_spec, modtool): Get the dependency tree of a toolchain :param toolchain_spec: toolchain spec to get the dependencies of + :param modtool: module tool used + :return: The dependency tree of the toolchain spec """ path = robot_find_easyconfig(toolchain_spec['name'], toolchain_spec['version']) @@ -750,6 +755,9 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): :param source_toolchain: initial toolchain of the easyconfig(s) :param target_toolchain: target toolchain for tweaked easyconfig(s) + :param modtool: module tool used + + :return: mapping from source hierarchy to target hierarchy """ tc_mapping = {} initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, require_capabilities=True) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 3e62ae17a0..f797aa0a6e 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -318,6 +318,8 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) + # build specs should not be passed down to resolved dependencies, + # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) diff --git a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb index 86888f1a69..c265058180 100644 --- a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.25.eb @@ -16,7 +16,6 @@ toolchain = {'name': 'dummy', 'version': ''} dependencies = [ ('GCCcore', version), # binutils built on top of GCCcore, which was built on top of (dummy-built) binutils - # excluded, good enough for testing purposes ('binutils', binutilsver, '', ('GCCcore', version)), ] diff --git a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb index 4ae3f5c7e9..e5204458bf 100644 --- a/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb +++ b/test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.9.3-2.26.eb @@ -16,7 +16,6 @@ toolchain = {'name': 'dummy', 'version': ''} dependencies = [ ('GCCcore', version), # binutils built on top of GCCcore, which was built on top of (dummy-built) binutils - # excluded, good enough for testing purposes ('binutils', binutilsver, '', ('GCCcore', version)), ] diff --git a/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb b/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb index 8d54862a1a..d2f022d78d 100644 --- a/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb +++ b/test/framework/easyconfigs/test_ecs/i/ictce/ictce-4.1.13.eb @@ -14,7 +14,7 @@ compver = '2011.13.367' fake_dependencies = [ ('icc', compver), ('ifort', compver), - ('impi', '4.1.3.049'), + ('impi', '4.1.0.027'), ('imkl', '10.3.12.361') ] diff --git a/test/framework/options.py b/test/framework/options.py index 6102d1a783..58c43ab22f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1446,14 +1446,16 @@ def test_try(self): test_cases = [ ([], 'toy/0.0'), - # --try-toolchain* has a different branch to all other try options, combining defaults back to regex + # combining --try-toolchain with other build options is too complicated, in this case the code defaults back + # to doing a simple regex substitution on the toolchain (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), # --try-toolchain is overridden by --toolchain (['--try-toolchain=gompi,1.3.12', '--toolchain=dummy,dummy'], 'toy/0.0'), (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-GCC-4.7.2'), - # --try-toolchain* has a different branch to all other try options, combining defaults back to regex + # combining --try-toolchain with other build options is too complicated, in this case the code defaults back + # to doing a simple regex substitution on the toolchain (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), # --try-amend is overridden by --amend @@ -1510,10 +1512,9 @@ def test_recursive_try(self): for extra_args in [[], ['--module-naming-scheme=HierarchicalMNS']]: outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) - # toolchain GCC/4.7.2 (subtoochain of gompi/1.4.10) should be listed (and present) - mark = 'x' + # toolchain GCC/4.7.2 (subtoolchain of gompi/1.4.10) should be listed (and present) - tc_regex = re.compile("^ \* \[%s\] .*/GCC-4.7.2.eb \(module: .*GCC/4.7.2\)$" % mark, re.M) + tc_regex = re.compile("^ \* \[x\] .*/GCC-4.7.2.eb \(module: .*GCC/4.7.2\)$", re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) # both toy and gzip dependency should be listed with gompi/1.4.10 toolchain diff --git a/test/framework/robot.py b/test/framework/robot.py index a80ce85c23..b1e47d846d 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -721,25 +721,69 @@ def test_get_toolchain_hierarchy(self): get_toolchain_hierarchy.clear() goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) - self.assertEqual(goolf_hierarchy, [ - {'name': 'GCC', 'version': '4.7.2', 'compiler_family': 'GCC', 'mpi_family': None, - 'lapack_family': None, 'blas_family': None, 'cuda': None}, - {'name': 'golf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': None, - 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': None}, - {'name': 'gompi', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', - 'lapack_family': None, 'blas_family': None, 'cuda': None}, - {'name': 'goolf', 'version': '1.4.10', 'compiler_family': 'GCC', 'mpi_family': 'OpenMPI', - 'lapack_family': 'OpenBLAS', 'blas_family': 'OpenBLAS', 'cuda': None}, - ]) + expected = [ + { + 'name': 'GCC', + 'version': '4.7.2', + 'comp_family': 'GCC', + 'mpi_family': None, + 'lapack_family': None, + 'blas_family': None, + 'cuda': None + }, + { + 'name': 'golf', + 'version': '1.4.10', + 'comp_family': 'GCC', + 'mpi_family': None, + 'lapack_family': 'OpenBLAS', + 'blas_family': 'OpenBLAS', + 'cuda': None + }, + { + 'name': 'gompi', + 'version': '1.4.10', + 'comp_family': 'GCC', + 'mpi_family': 'OpenMPI', + 'lapack_family': None, + 'blas_family': None, + 'cuda': None + }, + { + 'name': 'goolf', + 'version': '1.4.10', + 'comp_family': 'GCC', + 'mpi_family': 'OpenMPI', + 'lapack_family': 'OpenBLAS', + 'blas_family': 'OpenBLAS', + 'cuda': None + }, + ] + self.assertEqual(goolf_hierarchy, expected) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, require_capabilities=True) - self.assertEqual(iimpi_hierarchy, [ - {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': None, - 'lapack_family': None, 'blas_family': None, 'cuda': None}, - {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3', 'compiler_family': 'Intel', 'mpi_family': 'IntelMPI', - 'lapack_family': None, 'blas_family': None, 'cuda': None}, - ]) + expected = [ + { + 'name': 'iccifort', + 'version': '2013.5.192-GCC-4.8.3', + 'comp_family': 'Intel', + 'mpi_family': None, + 'lapack_family': None, + 'blas_family': None, + 'cuda': None + }, + { + 'name': 'iimpi', + 'version': '5.5.3-GCC-4.8.3', + 'comp_family': 'Intel', + 'mpi_family': 'IntelMPI', + 'lapack_family': None, + 'blas_family': None, + 'cuda': None + }, + ] + self.assertEqual(iimpi_hierarchy, expected) # test also including dummy init_config(build_options={ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 77cde8e926..98c7bdc2d0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -664,7 +664,8 @@ def test_toy_hierarchical(self): # test module paths/contents with goolf build extra_args = [ '--try-toolchain=goolf,1.4.10', - # Force using the regex method so try uses the full toolchain + # This test was created for the regex substitution of toolchains, to trigger this (rather than subtoolchain + # resolution) we must add an additional build option '--try-amend=parallel=1', ] self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index c41f9876dc..3b4d3dd6d4 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -161,8 +161,15 @@ def test_compare_toolchain_specs(self): iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, require_capabilities=True) - # Hierarchies are returned with top-level toolchain last, goolf has 3 elements here, intel has 2 - # goolf <-> iimpi (should return False) + # Hierarchies are returned with top-level toolchain last, goolf has 4 elements here, intel has 2 + self.assertEqual(goolf_hierarchy[0]['name'], 'GCC') + self.assertEqual(goolf_hierarchy[1]['name'], 'golf') + self.assertEqual(goolf_hierarchy[2]['name'], 'gompi') + self.assertEqual(goolf_hierarchy[3]['name'], 'goolf') + self.assertEqual(iimpi_hierarchy[0]['name'], 'iccifort') + self.assertEqual(iimpi_hierarchy[1]['name'], 'iimpi') + + # golf <-> iimpi (should return False) self.assertFalse(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs") # gompi <-> iimpi self.assertTrue(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1])) @@ -182,6 +189,14 @@ def test_match_minimum_tc_specs(self): goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, require_capabilities=True) + # Hierarchies are returned with top-level toolchain last, goolf has 4 elements here, intel has 2 + self.assertEqual(goolf_hierarchy[0]['name'], 'GCC') + self.assertEqual(goolf_hierarchy[1]['name'], 'golf') + self.assertEqual(goolf_hierarchy[2]['name'], 'gompi') + self.assertEqual(goolf_hierarchy[3]['name'], 'goolf') + self.assertEqual(iimpi_hierarchy[0]['name'], 'iccifort') + self.assertEqual(iimpi_hierarchy[1]['name'], 'iimpi') + # Compiler first self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], goolf_hierarchy), {'name': 'GCC', 'version': '4.7.2'}) @@ -208,12 +223,20 @@ def test_map_toolchain_hierarchies(self): gompi_tc = {'name': 'gompi', 'version': '1.4.10'} iimpi_tc = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} - self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc, self.modtool), - {'iccifort': {'name': 'GCC', 'version': '4.7.2'}, - 'iimpi': {'name': 'gompi', 'version': '1.4.10'}}) - self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), - {'GCC': {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, - 'gompi': {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}}) + # iccifort is mapped to GCC, iimpi is mapped to gompi + expected = { + 'iccifort': {'name': 'GCC', 'version': '4.7.2'}, + 'iimpi': {'name': 'gompi', 'version': '1.4.10'}, + } + self.assertEqual(map_toolchain_hierarchies(iimpi_tc, goolf_tc, self.modtool), expected) + + # GCC is mapped to iccifort, gompi is mapped to iimpi + expected = { + 'GCC': {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, + 'gompi': {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} + } + self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), expected) + # Expect an error when there is no possible mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies, @@ -222,10 +245,13 @@ def test_map_toolchain_hierarchies(self): # Test that we correctly include GCCcore binutils when it is there gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} - self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), - {'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, - 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, - 'binutils': {'version': '2.25', 'versionsuffix': ''}}) + # Should see a binutils in the mapping (2.26 will get mapped to 2.25) + expected = { + 'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, + 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, + 'binutils': {'version': '2.25', 'versionsuffix': ''} + } + self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected) def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" @@ -235,18 +261,22 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() + # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) - key, value = 'version', '2.25' - self.assertTrue(key in tweaked_dict['builddependencies'][0] and - value == tweaked_dict['builddependencies'][0][key]) + # Also check that binutils has been mapped + for key, value in {'name': 'binutils', 'version': '2.25'}.items(): + self.assertTrue(key in tweaked_dict['builddependencies'][0] and + value == tweaked_dict['builddependencies'][0][key]) def suite(): """ return all the tests in this file """ From 1f1b03de1289981b542256e61504f2f1eaf38c00 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:04:45 +0200 Subject: [PATCH 42/51] Address last of the comments --- easybuild/framework/easyconfig/easyconfig.py | 16 ++-- easybuild/framework/easyconfig/tweak.py | 81 ++++++++++---------- test/framework/robot.py | 4 +- test/framework/tweak.py | 30 +++++++- 4 files changed, 76 insertions(+), 55 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2f0a11312b..60aaf03e25 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -119,16 +119,16 @@ def toolchain_hierarchy_cache(func): cache = {} @functools.wraps(func) - def cache_aware_func(toolchain, require_capabilities=False): + def cache_aware_func(toolchain, incl_capabilities=False): """Look up toolchain hierarchy in cache first, determine and cache it if not available yet.""" - cache_key = (toolchain['name'], toolchain['version'], require_capabilities) + cache_key = (toolchain['name'], toolchain['version'], incl_capabilities) # fetch from cache if available, cache it if it's not if cache_key in cache: _log.debug("Using cache to return hierarchy for toolchain %s: %s", str(toolchain), cache[cache_key]) return cache[cache_key] else: - toolchain_hierarchy = func(toolchain, require_capabilities) + toolchain_hierarchy = func(toolchain, incl_capabilities) cache[cache_key] = toolchain_hierarchy return cache[cache_key] @@ -138,7 +138,7 @@ def cache_aware_func(toolchain, require_capabilities=False): return cache_aware_func -def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, require_capabilities=False): +def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=False): """ Returns unique version for subtoolchain, in tc dict. If there is no unique version: @@ -155,7 +155,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, # dummy toolchain: bottom of the hierarchy if subtoolchain_name == DUMMY_TOOLCHAIN_NAME: - if build_option('add_dummy_to_minimal_toolchains') and not require_capabilities: + if build_option('add_dummy_to_minimal_toolchains') and not incl_capabilities: subtoolchain_version = '' elif len(uniq_subtc_versions) == 1: subtoolchain_version = list(uniq_subtc_versions)[0] @@ -172,7 +172,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, @toolchain_hierarchy_cache -def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): +def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. @@ -273,7 +273,7 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): for subtoolchain_name in subtoolchain_names: subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, - require_capabilities=require_capabilities) + incl_capabilities=incl_capabilities) # add to hierarchy and move to next if subtoolchain_version is not None and subtoolchain_name not in visited: tc = {'name': subtoolchain_name, 'version': subtoolchain_version} @@ -282,7 +282,7 @@ def get_toolchain_hierarchy(parent_toolchain, require_capabilities=False): visited.add(subtoolchain_name) # also add toolchain capabilities - if require_capabilities: + if incl_capabilities: for toolchain in toolchain_hierarchy: toolchain_class, _ = search_toolchain(toolchain['name']) tc = toolchain_class(version=toolchain['version']) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 82fa2e535b..2ce6bb6232 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -45,6 +45,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS, CAPABILITIES +from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -713,8 +714,7 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): if compare_toolchain_specs(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used in the target if the original toolchain was also # GCCcore - if target_tc_spec['name'] != GCCcore.NAME or \ - (source_tc_spec['name'] == GCCcore.NAME and target_tc_spec['name'] == GCCcore.NAME): + if target_tc_spec['name'] != GCCcore.NAME or source_tc_spec['name'] == GCCcore.NAME: minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['comp_family'] break @@ -733,7 +733,7 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): def get_dep_tree_of_toolchain(toolchain_spec, modtool): """ - Get the dependency tree of a toolchain + Get list of dependencies of a toolchain (as EasyConfig objects) :param toolchain_spec: toolchain spec to get the dependencies of :param modtool: module tool used @@ -746,7 +746,7 @@ def get_dep_tree_of_toolchain(toolchain_spec, modtool): toolchain_spec['name'], toolchain_spec['version']) ec = process_easyconfig(path, validate=False) - return resolve_dependencies(ec, modtool) + return [dep['ec'] for dep in resolve_dependencies(ec, modtool)] def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): @@ -760,35 +760,34 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): :return: mapping from source hierarchy to target hierarchy """ tc_mapping = {} - initial_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, require_capabilities=True) - target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, require_capabilities=True) + source_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, incl_capabilities=True) + target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, incl_capabilities=True) - for toolchain_spec in initial_tc_hierarchy: + for toolchain_spec in source_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) # Check for presence of binutils in source and target toolchain dependency trees (only do this when GCCcore is # present in both and GCCcore is not the top of the tree) - if GCCcore.NAME in [tc_spec['name'] for tc_spec in initial_tc_hierarchy]\ - and GCCcore.NAME in [tc_spec['name'] for tc_spec in target_tc_hierarchy]\ - and initial_tc_hierarchy[-1]['name'] != GCCcore.NAME: - + gcccore = GCCcore.NAME + source_tc_names = [tc_spec['name'] for tc_spec in source_tc_hierarchy] + target_tc_names = [tc_spec['name'] for tc_spec in target_tc_hierarchy] + if gcccore in source_tc_names and gcccore in target_tc_names and source_tc_hierarchy[-1]['name'] != gcccore: binutils = 'binutils' # Determine the dependency trees - source_dep_tree = get_dep_tree_of_toolchain(initial_tc_hierarchy[-1], modtool) + source_dep_tree = get_dep_tree_of_toolchain(source_tc_hierarchy[-1], modtool) target_dep_tree = get_dep_tree_of_toolchain(target_tc_hierarchy[-1], modtool) # Find the binutils mapping - if binutils in [dep['ec']['name'] for dep in source_dep_tree]: + if binutils in [dep['name'] for dep in source_dep_tree]: # We need the binutils that was built using GCCcore (we assume that everything is using standard behaviour: # build binutils with GCCcore and then use that for anything built with GCCcore) - binutils_list = [{'version': dep['ec']['version'], 'versionsuffix': dep['ec']['versionsuffix']} - for dep in target_dep_tree if (dep['ec']['name'] == binutils) and - (dep['ec']['toolchain']['name'] == GCCcore.NAME)] - # There should be one element in this list - if len(binutils_list) != 1: - raise EasyBuildError("Target hierarchy %s should have binutils using GCCcore, can't determine mapping!" - % target_tc_hierarchy[-1]) + binutils_deps = [dep for dep in target_dep_tree if dep['name'] == binutils] + binutils_gcccore_deps = [dep for dep in binutils_deps if dep['toolchain']['name'] == gcccore] + if len(binutils_gcccore_deps) == 1: + tc_mapping[binutils] = {'version': binutils_gcccore_deps[0]['version'], + 'versionsuffix': binutils_gcccore_deps[0]['versionsuffix']} else: - tc_mapping[binutils] = binutils_list[0] + raise EasyBuildError("Target hierarchy %s should have binutils using GCCcore, can't determine mapping!", + target_tc_hierarchy[-1]) return tc_mapping @@ -797,19 +796,23 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out - :param ec_spec: Location of original easyconfig - :param toolchain_mapping: Mapping between toolchain and target toolchain and target toolchain - :param targetdir: - :return mapped_spec: + :param ec_spec: Location of original easyconfig file + :param toolchain_mapping: Mapping between source toolchain and target toolchain + :param targetdir: Directory to dump the modified easyconfig file in + + :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] # Replace the toolchain if the mapping exists - if parsed_ec['ec']['toolchain']['name'] in toolchain_mapping: - parsed_ec['ec']['toolchain'] = toolchain_mapping[parsed_ec['ec']['toolchain']['name']] + tc_name = parsed_ec['ec']['toolchain']['name'] + if tc_name in toolchain_mapping: + new_toolchain = toolchain_mapping[tc_name] + _log.debug("Replacing parent toolchain %s with %s", parsed_ec['ec']['toolchain'], new_toolchain) + parsed_ec['ec']['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies filter_deps = build_option('filter_deps') - for key in ['builddependencies', 'dependencies', 'hiddendependencies']: + for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... for idx, dep in enumerate(parsed_ec['ec'][key]): @@ -818,22 +821,18 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # skip dependencies that are marked as external modules if dep['external_module']: continue - if dep['toolchain']['name'] in toolchain_mapping: - orig_dep['toolchain'] = toolchain_mapping[dep['toolchain']['name']] + dep_tc_name = dep['toolchain']['name'] + if dep_tc_name in toolchain_mapping: + orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] # Replace the binutils version (if necessary) - if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and - dep['toolchain']['name'] == GCCcore.NAME): - orig_dep['version'] = toolchain_mapping['binutils']['version'] - orig_dep['versionsuffix'] = toolchain_mapping['binutils']['versionsuffix'] - if not dep['external_module']: - # set module names - orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) - orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): + orig_dep.update(toolchain_mapping['binutils']) + # set module names + orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) + orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) - if targetdir is None: - targetdir = tempfile.gettempdir() - tweaked_spec = os.path.join(targetdir, ec_filename) + tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) parsed_ec['ec'].dump(tweaked_spec) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) diff --git a/test/framework/robot.py b/test/framework/robot.py index b1e47d846d..b80b97588c 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -720,7 +720,7 @@ def test_get_toolchain_hierarchy(self): get_toolchain_hierarchy.clear() - goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, incl_capabilities=True) expected = [ { 'name': 'GCC', @@ -762,7 +762,7 @@ def test_get_toolchain_hierarchy(self): self.assertEqual(goolf_hierarchy, expected) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, - require_capabilities=True) + incl_capabilities=True) expected = [ { 'name': 'iccifort', diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 3b4d3dd6d4..ae919dd196 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -36,6 +36,7 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs +from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies, map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes @@ -157,9 +158,9 @@ def test_compare_toolchain_specs(self): 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() - goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, incl_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, - require_capabilities=True) + incl_capabilities=True) # Hierarchies are returned with top-level toolchain last, goolf has 4 elements here, intel has 2 self.assertEqual(goolf_hierarchy[0]['name'], 'GCC') @@ -186,9 +187,9 @@ def test_match_minimum_tc_specs(self): 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() - goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, require_capabilities=True) + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}, incl_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, - require_capabilities=True) + incl_capabilities=True) # Hierarchies are returned with top-level toolchain last, goolf has 4 elements here, intel has 2 self.assertEqual(goolf_hierarchy[0]['name'], 'GCC') self.assertEqual(goolf_hierarchy[1]['name'], 'golf') @@ -211,6 +212,27 @@ def test_match_minimum_tc_specs(self): self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs, goolf_hierarchy[3], iimpi_hierarchy) + def test_dep_tree_of_toolchain(self): + """Test getting list of dependencies of a toolchain (as EasyConfig objects)""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': test_easyconfigs, + }) + toolchain_spec = {'name': 'goolf', 'version': '1.4.10'} + list_of_deps = get_dep_tree_of_toolchain(toolchain_spec, self.modtool) + expected_deps = [ + ['OpenBLAS', '0.2.6'], + ['hwloc', '1.6.2'], + ['OpenMPI', '1.6.4'], + ['gompi', '1.4.10'], + ['FFTW', '3.3.3'], + ['ScaLAPACK', '2.0.2'], + ['goolf', '1.4.10'] + ] + actual_deps = [[dep['name'], dep['version']] for dep in list_of_deps] + self.assertItemsEqual(expected_deps, actual_deps) + def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 0965f9a568c28ec27bce55c11e08f14d63cb5f0e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:12:31 +0200 Subject: [PATCH 43/51] Address newer comments --- easybuild/framework/easyconfig/easyconfig.py | 13 +++++-------- easybuild/framework/easyconfig/tweak.py | 3 ++- easybuild/tools/toolchain/toolchain.py | 2 ++ test/framework/tweak.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 60aaf03e25..9d8cc9d233 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -68,6 +68,7 @@ from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.systemtools import check_os_dependency from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION +from easybuild.tools.toolchain.toolchain import CAPABILITIES from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars @@ -82,9 +83,6 @@ # name of easyconfigs archive subdirectory EASYCONFIGS_ARCHIVE_DIR = '__archive__' -# Available capabilities of toolchains -CAPABILITIES = ['comp_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda'] - try: import autopep8 @@ -202,8 +200,8 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) - # the parent toolchain is at the top of the hierarchy, we need a copy so that adding capabilities (below) doesn't - # affect the original object + # the parent toolchain is at the top of the hierarchy, + # we need a copy so that adding capabilities (below) doesn't affect the original object toolchain_hierarchy = [copy.copy(parent_toolchain)] # use a queue to handle a breadth-first-search of the hierarchy, # which is required to take into account the potential for multiple subtoolchains @@ -291,9 +289,8 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): if capability == 'cuda': # use None rather than False, useful to have it consistent with the rest toolchain['cuda'] = ('CUDA_CC' in tc.variables) or None - else: - if hasattr(tc, capability): - toolchain[capability] = getattr(tc, capability)() + elif hasattr(tc, capability): + toolchain[capability] = getattr(tc, capability)() _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 2ce6bb6232..d854662ec3 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -44,7 +44,7 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS, CAPABILITIES +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning @@ -53,6 +53,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.toolchain.toolchain import CAPABILITIES from easybuild.tools.utilities import quote_str diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 0ca244d131..740b346cf3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -60,6 +60,8 @@ RPATH_WRAPPERS_SUBDIR = 'rpath_wrappers' +# Available capabilities of toolchains +CAPABILITIES = ['comp_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda'] class Toolchain(object): """General toolchain class""" diff --git a/test/framework/tweak.py b/test/framework/tweak.py index ae919dd196..6b64cb84a2 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -296,7 +296,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped - for key, value in {'name': 'binutils', 'version': '2.25'}.items(): + for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key]) From 5377ff8f0aed05bb462508836226d97829752e21 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:24:52 +0200 Subject: [PATCH 44/51] Fix comments --- easybuild/framework/easyconfig/tweak.py | 1 - test/framework/options.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index d854662ec3..6eaab6a9da 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -174,7 +174,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) if new_ec_file: - # Need to update the toolchain the build_specs to match the toolchain mapping new_ecs = process_easyconfig(new_ec_file, build_specs=verification_build_specs) tweaked_easyconfigs.extend(new_ecs) else: diff --git a/test/framework/options.py b/test/framework/options.py index 58c43ab22f..7d5f23f8d0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -851,6 +851,8 @@ def test_try_robot_force(self): ("GCC-4.6.4.eb", "GCC/4.6.4", 'x'), ("OpenMPI-1.6.4-GCC-4.6.4.eb", "OpenMPI/1.6.4-GCC-4.6.4", 'x'), # OpenBLAS dependency is listed, but not there => ' ' + # toolchain used for OpenBLAS is mapped to GCC/4.6.4 subtoolchain in gompi/1.3.12 + # (rather than the original GCC/4.7.2 as subtoolchain of gompi/1.4.10) ("OpenBLAS-0.2.6-GCC-4.6.4-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-GCC-4.6.4-LAPACK-3.4.2", ' '), # both FFTW and ScaLAPACK are listed => 'F' ("ScaLAPACK-%s.eb" % scalapack_ver, "ScaLAPACK/%s" % scalapack_ver, 'F'), From 61a0bcb26f907e31866628999316d5e4877632eb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:38:04 +0200 Subject: [PATCH 45/51] Fix more comments --- test/framework/options.py | 4 +++- test/framework/toy_build.py | 13 ++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 7d5f23f8d0..88edabf28d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1519,7 +1519,9 @@ def test_recursive_try(self): tc_regex = re.compile("^ \* \[x\] .*/GCC-4.7.2.eb \(module: .*GCC/4.7.2\)$", re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) - # both toy and gzip dependency should be listed with gompi/1.4.10 toolchain + # both toy and gzip dependency should be listed with new toolchains + # in this case we map original toolchain `dummy` to the compiler-only GCC subtoolchain of gompi/1.4.10 + # since this subtoolchain already has sufficient capabilities (we do not map higher than necessary) for ec_name in ['gzip-1.4', 'toy-0.0']: ec = '%s-GCC-4.7.2.eb' % ec_name if extra_args: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 98c7bdc2d0..d0a1c4afb5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -865,9 +865,9 @@ def test_toy_hierarchical_subdir_user_modules(self): toy_modtxt = read_file(toy_mod) # No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary - # for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - # regex = re.compile('load.*' + modname, re.M) - # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + regex = re.compile('load.*' + modname, re.M) + self.assertFalse(regex.search(toy_modtxt), "Pattern '%s' not found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) @@ -915,10 +915,9 @@ def test_toy_hierarchical_subdir_user_modules(self): toy_modtxt = read_file(toy_mod) # No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary - - # for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: - # regex = re.compile('load.*' + modname, re.M) - # self.assertTrue(regex.search(toy_modtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_modtxt)) + for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: + regex = re.compile('load.*' + modname, re.M) + self.assertFalse(regex.search(toy_modtxt), "Pattern '%s' not found in: %s" % (regex.pattern, toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) From 14d80b7f8dbea9fa8e569642a1f0f73f2d1f4aaf Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:39:47 +0200 Subject: [PATCH 46/51] Appease the hound --- test/framework/toy_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d0a1c4afb5..9b7e59be5c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -917,7 +917,8 @@ def test_toy_hierarchical_subdir_user_modules(self): # No math libs in original toolchain, --try-toolchain is too clever to upgrade it beyond necessary for modname in ['FFTW', 'OpenBLAS', 'ScaLAPACK']: regex = re.compile('load.*' + modname, re.M) - self.assertFalse(regex.search(toy_modtxt), "Pattern '%s' not found in: %s" % (regex.pattern, toy_modtxt)) + self.assertFalse(regex.search(toy_modtxt), "Pattern '%s' not found in: %s" % (regex.pattern, + toy_modtxt)) for modname in ['GCC', 'OpenMPI']: regex = re.compile('load.*' + modname, re.M) From 4d8b5030f1652bed0140504064acd1e6dad51fff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:49:39 +0200 Subject: [PATCH 47/51] Address more comments --- easybuild/framework/easyconfig/tweak.py | 6 +++--- test/framework/tweak.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6eaab6a9da..07c28dcaff 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -680,9 +680,9 @@ def obtain_ec_for(specs, paths, fp=None): raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.") -def compare_toolchain_specs(source_tc_spec, target_tc_spec): +def check_capability_mapping(source_tc_spec, target_tc_spec): """ - Compare whether a source toolchain is mappable to a target toolchain + Compare whether the capabilities of a source toolchain are all present in a target toolchain :param source_tc_spec: specs of source toolchain :param target_tc_spec: specs of target toolchain @@ -711,7 +711,7 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): # break out once we've found the first match since the hierarchy is ordered low to high in terms of capabilities for target_tc_spec in target_tc_hierarchy: - if compare_toolchain_specs(source_tc_spec, target_tc_spec): + if check_capability_mapping(source_tc_spec, target_tc_spec): # GCCcore has compiler capabilities but should only be used in the target if the original toolchain was also # GCCcore if target_tc_spec['name'] != GCCcore.NAME or source_tc_spec['name'] == GCCcore.NAME: diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 6b64cb84a2..2397133e11 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -35,7 +35,7 @@ from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one -from easybuild.framework.easyconfig.tweak import compare_toolchain_specs, match_minimum_tc_specs +from easybuild.framework.easyconfig.tweak import check_capability_mapping, match_minimum_tc_specs from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies, map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError @@ -150,7 +150,7 @@ def test_tweak_one_version(self): tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') - def test_compare_toolchain_specs(self): + def test_check_capability_mapping(self): """Test comparing the functionality of two toolchains""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ @@ -171,13 +171,13 @@ def test_compare_toolchain_specs(self): self.assertEqual(iimpi_hierarchy[1]['name'], 'iimpi') # golf <-> iimpi (should return False) - self.assertFalse(compare_toolchain_specs(goolf_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs") + self.assertFalse(check_capability_mapping(goolf_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs") # gompi <-> iimpi - self.assertTrue(compare_toolchain_specs(goolf_hierarchy[2], iimpi_hierarchy[1])) + self.assertTrue(check_capability_mapping(goolf_hierarchy[2], iimpi_hierarchy[1])) # GCC <-> iimpi - self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[1])) + self.assertTrue(check_capability_mapping(goolf_hierarchy[0], iimpi_hierarchy[1])) # GCC <-> iccifort - self.assertTrue(compare_toolchain_specs(goolf_hierarchy[0], iimpi_hierarchy[0])) + self.assertTrue(check_capability_mapping(goolf_hierarchy[0], iimpi_hierarchy[0])) def test_match_minimum_tc_specs(self): """Test matching a toolchain to lowest possible in a hierarchy""" From 21a638ce880e99747110682989f397b161d2baa5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 10:54:37 +0200 Subject: [PATCH 48/51] Fix broken test --- test/framework/tweak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2397133e11..c535501728 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -218,6 +218,7 @@ def test_dep_tree_of_toolchain(self): init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, + 'check_osdeps': False, }) toolchain_spec = {'name': 'goolf', 'version': '1.4.10'} list_of_deps = get_dep_tree_of_toolchain(toolchain_spec, self.modtool) From 07384d5cb6d9577caec75de1009e85538ba81319 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Sep 2018 12:13:15 +0200 Subject: [PATCH 49/51] Use more general test method --- test/framework/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index c535501728..427fdf2484 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -232,7 +232,7 @@ def test_dep_tree_of_toolchain(self): ['goolf', '1.4.10'] ] actual_deps = [[dep['name'], dep['version']] for dep in list_of_deps] - self.assertItemsEqual(expected_deps, actual_deps) + self.assertEqual(expected_deps, actual_deps) def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" From 5569c50bacfafded14be977f03ef4f883d9a2c13 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 14 Sep 2018 12:26:44 +0200 Subject: [PATCH 50/51] Update tweak.py --- easybuild/framework/easyconfig/tweak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 07c28dcaff..a5170853d9 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -32,6 +32,7 @@ :author: Jens Timmerman (Ghent University) :author: Toon Willems (Ghent University) :author: Fotis Georgatos (Uni.Lu, NTUA) +:author: Alan O'Cais (Juelich Supercomputing Centre) """ import copy import glob From 67b816e9131488a485296ed0319f26b375639670 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Sep 2018 11:13:00 +0200 Subject: [PATCH 51/51] minor code cleanup --- easybuild/framework/easyconfig/easyconfig.py | 15 ++--- easybuild/framework/easyconfig/tweak.py | 60 ++++++++++---------- easybuild/tools/toolchain/toolchain.py | 28 ++++++--- 3 files changed, 57 insertions(+), 46 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 88ded438ad..47e761ce18 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -68,7 +68,7 @@ from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.systemtools import check_os_dependency from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION -from easybuild.tools.toolchain.toolchain import CAPABILITIES +from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars @@ -145,8 +145,6 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, optional toolchains or dummy without add_dummy_to_minimal_toolchains. * in all other cases, raises an exception. """ - current_tc_name, current_tc_version = current_tc['name'], current_tc['version'] - uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) # init with "skipped" subtoolchain_version = None @@ -161,10 +159,10 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, if subtoolchain_name not in optional_toolchains: # raise error if the subtoolchain considered now is not optional raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", - subtoolchain_name, current_tc_name) + subtoolchain_name, current_tc['name']) else: raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", - subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions))) + subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions))) return subtoolchain_version @@ -284,15 +282,14 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): for toolchain in toolchain_hierarchy: toolchain_class, _ = search_toolchain(toolchain['name']) tc = toolchain_class(version=toolchain['version']) - for capability in CAPABILITIES: + for capability in TOOLCHAIN_CAPABILITIES: # cuda is the special case which doesn't have a family attribute - if capability == 'cuda': + if capability == TOOLCHAIN_CAPABILITY_CUDA: # use None rather than False, useful to have it consistent with the rest - toolchain['cuda'] = ('CUDA_CC' in tc.variables) or None + toolchain[capability] = ('CUDA_CC' in tc.variables) or None elif hasattr(tc, capability): toolchain[capability] = getattr(tc, capability)() - _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a5170853d9..0beda4b3f7 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -54,7 +54,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME -from easybuild.tools.toolchain.toolchain import CAPABILITIES +from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES from easybuild.tools.utilities import quote_str @@ -98,14 +98,14 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported - for key in keys: - if key not in ['toolchain_name', 'toolchain_version', 'toolchain']: - print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") - revert_to_regex = True + if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain'] for key in keys): + print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") + revert_to_regex = True if not revert_to_regex: - # We're doing something with the toolchain, build specifications should be applied to whole dependency graph - # Obtain full dependency graph for specified easyconfigs + # we're doing something with the toolchain, + # so build specifications should be applied to whole dependency graph; + # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) modifying_toolchains = True @@ -113,6 +113,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain['name'] = build_specs['toolchain_name'] else: target_toolchain['name'] = source_toolchain['name'] + if 'toolchain_version' in keys: target_toolchain['version'] = build_specs['toolchain_version'] else: @@ -120,16 +121,17 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) - _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) + _log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs) orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) # We can leave any dependencies they may have as they will only be used if required (or originally listed) - _log.debug("Filtering out toolchain hierarchy for %s" % source_toolchain) + _log.debug("Filtering out toolchain hierarchy for %s", source_toolchain) i = 0 while i < len(orig_ecs): - if orig_ecs[i]['ec']['name'] in [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]: + tc_names = [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)] + if orig_ecs[i]['ec']['name'] in tc_names: # drop elements in toolchain hierarchy del orig_ecs[i] else: @@ -149,38 +151,38 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: - # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use - # the prepended path so that they are found first). + # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line + # (and use the prepended path so that they are found first). # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot # either from existing easyconfigs or, if that fails, from easyconfigs in the appended path - new_ec_file = '' - verification_build_specs = dict(build_specs) + tc_name = orig_ec['ec']['toolchain']['name'] + + new_ec_file = None + verification_build_specs = copy.copy(build_specs) if orig_ec['spec'] in listed_ec_paths: if modifying_toolchains: - if orig_ec['ec']['toolchain']['name'] in src_to_dst_tc_mapping: + if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, tweaked_ecs_path) # Need to update the toolchain in the build_specs to match the toolchain mapping keys = verification_build_specs.keys() if 'toolchain_name' in keys: - verification_build_specs['toolchain_name'] = \ - src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]['name'] + verification_build_specs['toolchain_name'] = src_to_dst_tc_mapping[tc_name]['name'] if 'toolchain_version' in keys: - verification_build_specs['toolchain_version'] = \ - src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']]['version'] + verification_build_specs['toolchain_version'] = src_to_dst_tc_mapping[tc_name]['version'] if 'toolchain' in keys: - verification_build_specs['toolchain'] = \ - src_to_dst_tc_mapping[orig_ec['ec']['toolchain']['name']] + verification_build_specs['toolchain'] = src_to_dst_tc_mapping[tc_name] else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path) + if new_ec_file: new_ecs = process_easyconfig(new_ec_file, build_specs=verification_build_specs) tweaked_easyconfigs.extend(new_ecs) else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path if modifying_toolchains: - if orig_ec['ec']['toolchain']['name'] in src_to_dst_tc_mapping: + if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, targetdir=tweaked_ecs_deps_path) else: @@ -562,7 +564,7 @@ def unique(l): versuff = None other_params = {'versionprefix': None, 'versionsuffix': None} for (param, val) in specs.items(): - if not param in handled_params: + if param not in handled_params: other_params.update({param: val}) _log.debug("Filtering based on other parameters (specified via --amend): %s" % other_params) @@ -692,7 +694,7 @@ def check_capability_mapping(source_tc_spec, target_tc_spec): """ can_map = True # Check they have same capabilities - for key in CAPABILITIES: + for key in TOOLCHAIN_CAPABILITIES: if target_tc_spec[key] is None and source_tc_spec[key] is not None: can_map = False break @@ -713,8 +715,8 @@ def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy): # break out once we've found the first match since the hierarchy is ordered low to high in terms of capabilities for target_tc_spec in target_tc_hierarchy: if check_capability_mapping(source_tc_spec, target_tc_spec): - # GCCcore has compiler capabilities but should only be used in the target if the original toolchain was also - # GCCcore + # GCCcore has compiler capabilities, + # but should only be used in the target if the original toolchain was also GCCcore if target_tc_spec['name'] != GCCcore.NAME or source_tc_spec['name'] == GCCcore.NAME: minimal_matching_toolchain = {'name': target_tc_spec['name'], 'version': target_tc_spec['version']} target_compiler_family = target_tc_spec['comp_family'] @@ -767,8 +769,8 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): for toolchain_spec in source_tc_hierarchy: tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy) - # Check for presence of binutils in source and target toolchain dependency trees (only do this when GCCcore is - # present in both and GCCcore is not the top of the tree) + # Check for presence of binutils in source and target toolchain dependency trees + # (only do this when GCCcore is present in both and GCCcore is not the top of the tree) gcccore = GCCcore.NAME source_tc_names = [tc_spec['name'] for tc_spec in source_tc_hierarchy] target_tc_names = [tc_spec['name'] for tc_spec in target_tc_hierarchy] @@ -811,8 +813,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= new_toolchain = toolchain_mapping[tc_name] _log.debug("Replacing parent toolchain %s with %s", parsed_ec['ec']['toolchain'], new_toolchain) parsed_ec['ec']['toolchain'] = new_toolchain + # Replace the toolchains of all the dependencies - filter_deps = build_option('filter_deps') for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 740b346cf3..91bec43645 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -38,11 +38,10 @@ from vsc.utils import fancylogger from vsc.utils.missing import nub -import easybuild.tools.toolchain from easybuild.tools.build_log import EasyBuildError, dry_run_msg from easybuild.tools.config import build_option, install_path from easybuild.tools.environment import setvar -from easybuild.tools.filetools import adjust_permissions, find_eb_script, mkdir, read_file, which, write_file +from easybuild.tools.filetools import adjust_permissions, find_eb_script, read_file, which, write_file from easybuild.tools.module_generator import dependencies_for from easybuild.tools.modules import get_software_root, get_software_root_env_var_name from easybuild.tools.modules import get_software_version, get_software_version_env_var_name @@ -60,8 +59,21 @@ RPATH_WRAPPERS_SUBDIR = 'rpath_wrappers' -# Available capabilities of toolchains -CAPABILITIES = ['comp_family', 'mpi_family', 'blas_family', 'lapack_family', 'cuda'] +# available capabilities of toolchains +# values match method names supported by Toolchain class (except for 'cuda') +TOOLCHAIN_CAPABILITY_BLAS_FAMILY = 'blas_family' +TOOLCHAIN_CAPABILITY_COMP_FAMILY = 'comp_family' +TOOLCHAIN_CAPABILITY_CUDA = 'cuda' +TOOLCHAIN_CAPABILITY_LAPACK_FAMILY = 'lapack_family' +TOOLCHAIN_CAPABILITY_MPI_FAMILY = 'mpi_family' +TOOLCHAIN_CAPABILITIES = [ + TOOLCHAIN_CAPABILITY_BLAS_FAMILY, + TOOLCHAIN_CAPABILITY_COMP_FAMILY, + TOOLCHAIN_CAPABILITY_CUDA, + TOOLCHAIN_CAPABILITY_LAPACK_FAMILY, + TOOLCHAIN_CAPABILITY_MPI_FAMILY, +] + class Toolchain(object): """General toolchain class""" @@ -651,7 +663,7 @@ def compilers(self): if self.name == DUMMY_TOOLCHAIN_NAME: c_comps = ['gcc', 'g++'] - fortran_comps = ['gfortran'] + fortran_comps = ['gfortran'] else: c_comps = [self.COMPILER_CC, self.COMPILER_CXX] fortran_comps = [self.COMPILER_F77, self.COMPILER_F90, self.COMPILER_FC] @@ -687,7 +699,7 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No # set the variables # onlymod can be comma-separated string of variables not to be set - if onlymod == True: + if onlymod is True: self.log.debug("prepare: do not set additional variables onlymod=%s", onlymod) self.generate_vars() else: @@ -855,11 +867,11 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): if cpp is not None: for p in cpp: - if not p in cpp_paths: + if p not in cpp_paths: cpp_paths.append(p) if ld is not None: for p in ld: - if not p in ld_paths: + if p not in ld_paths: ld_paths.append(p) if not names: