diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index cccf0bc202..94f81c9d9c 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -28,11 +28,13 @@ :author: Denis Kristak (Inuits) :author: Pavel Grochal (Inuits) :author: Kenneth Hoste (HPC-UGent) +:author: Caspar van Leeuwen (SURF) """ +import pprint + from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import only_if_module_is_available try: @@ -69,30 +71,13 @@ class EasyStack(object): def __init__(self): self.easybuild_version = None self.robot = False - self.software_list = [] - self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension - # A dict where keys are easyconfig names, values are dictionary of options that should be - # applied for that easyconfig - self.ec_opts = {} - - def compose_ec_filenames(self): - """Returns a list of all easyconfig names""" - ec_filenames = [] - - # entries specified via 'software' top-level key - for sw in self.software_list: - full_ec_version = det_full_ec_version({ - 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, - 'version': sw.version, - 'versionsuffix': sw.versionsuffix, - }) - ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) - ec_filenames.append(ec_filename) - - # entries specified via 'easyconfigs' top-level key - for ec in self.easyconfigs: - ec_filenames.append(ec) - return ec_filenames + self.ec_opt_tuples = [] # A list of tuples (easyconfig_name, eaysconfig_specific_opts) + + def __str__(self): + """ + Pretty printing of an EasyStack instance + """ + return pprint.pformat(self.ec_opt_tuples) # flags applicable to all sw (i.e. robot) def get_general_options(self): @@ -129,37 +114,33 @@ def parse(filepath): except yaml.YAMLError as err: raise EasyBuildError("Failed to parse %s: %s" % (filepath, err)) - easystack_data = None - top_keys = ('easyconfigs', 'software') - keys_found = [] - for key in top_keys: - if key in easystack_raw: - keys_found.append(key) - # For now, we don't support mixing multiple top_keys, so check that only one was defined - if len(keys_found) > 1: - keys_string = ', '.join(keys_found) - msg = "Specifying multiple top level keys (%s) " % keys_string - msg += "in one EasyStack file is currently not supported" - msg += ", see %s for documentation." % EASYSTACK_DOC_URL - raise EasyBuildError(msg) - elif len(keys_found) == 0: - msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" - msg += ", see %s for documentation." % EASYSTACK_DOC_URL - raise EasyBuildError(msg) - else: - key = keys_found[0] + key = 'easyconfigs' + if key in easystack_raw: easystack_data = easystack_raw[key] - - parse_method_name = 'parse_by_' + key - parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None) - if parse_method is None: - raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name) + if isinstance(easystack_data, dict) or isinstance(easystack_data, str): + datatype = 'dict' if isinstance(easystack_data, dict) else 'str' + msg = '\n'.join([ + "Found %s value for '%s' in %s, should be list." % (datatype, key, filepath), + "Make sure you use '-' to create list items under '%s', for example:" % key, + " easyconfigs:", + " - example-1.0.eb", + " - example-2.0.eb:", + " options:" + " ...", + ]) + raise EasyBuildError(msg) + elif not isinstance(easystack_data, list): + raise EasyBuildError("Value type for '%s' in %s should be list, found %s", + key, filepath, type(easystack_data)) + else: + raise EasyBuildError("Top-level key '%s' missing in easystack file %s", key, filepath) # assign general easystack attributes easybuild_version = easystack_raw.get('easybuild_version', None) robot = easystack_raw.get('robot', False) - return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot) + return EasyStackParser.parse_by_easyconfigs(filepath, easystack_data, + easybuild_version=easybuild_version, robot=robot) @staticmethod def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False): @@ -173,7 +154,7 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa if isinstance(easyconfig, str): if not easyconfig.endswith('.eb'): easyconfig = easyconfig + '.eb' - easystack.easyconfigs.append(easyconfig) + easystack.ec_opt_tuples.append((easyconfig, None)) elif isinstance(easyconfig, dict): if len(easyconfig) == 1: # Get single key from dictionary 'easyconfig' @@ -183,12 +164,18 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa easyconf_name_with_eb = easyconf_name + '.eb' else: easyconf_name_with_eb = easyconf_name - easystack.easyconfigs.append(easyconf_name_with_eb) - # Add options to the ec_opts dict - if 'options' in easyconfig[easyconf_name].keys(): - easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options'] + # Get options + ec_dict = easyconfig[easyconf_name] or {} + + # make sure only 'options' key is used (for now) + if any(x != 'options' for x in ec_dict): + msg = "Found one or more invalid keys for %s (only 'options' supported): %s" + raise EasyBuildError(msg, easyconf_name, ', '.join(sorted(ec_dict.keys()))) + + opts = ec_dict.get('options') + easystack.ec_opt_tuples.append((easyconf_name_with_eb, opts)) else: - dict_keys = ', '.join(easyconfig.keys()) + dict_keys = ', '.join(sorted(easyconfig.keys())) msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), " msg += "instead found keys: %s" % dict_keys msg += ", see %s for documentation." % EASYSTACK_DOC_URL @@ -196,119 +183,6 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa return easystack - @staticmethod - def parse_by_software(filepath, software, easybuild_version=None, robot=False): - """ - Parse easystack file with 'software' as top-level key. - """ - - easystack = EasyStack() - - # assign software-specific easystack attributes - for name in software: - # ensure we have a string value (YAML parser returns type = dict - # if levels under the current attribute are present) - check_value(name, "software name") - try: - toolchains = software[name]['toolchains'] - except KeyError: - raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) - for toolchain in toolchains: - check_value(toolchain, "software %s" % name) - - if toolchain == 'SYSTEM': - toolchain_name, toolchain_version = 'system', '' - else: - toolchain_parts = toolchain.split('-', 1) - if len(toolchain_parts) == 2: - toolchain_name, toolchain_version = toolchain_parts - elif len(toolchain_parts) == 1: - toolchain_name, toolchain_version = toolchain, '' - else: - raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", - name, filepath, toolchain_parts) - - try: - # if version string containts asterisk or labels, raise error (asterisks not supported) - versions = toolchains[toolchain]['versions'] - except TypeError as err: - wrong_structure_err = "An error occurred when interpreting " - wrong_structure_err += "the data for software %s: %s" % (name, err) - raise EasyBuildError(wrong_structure_err) - if '*' in str(versions): - asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. " - asterisk_err += "Wildcard feature is not supported yet." - raise EasyBuildError(asterisk_err, name, filepath) - - # yaml versions can be in different formats in yaml file - # firstly, check if versions in yaml file are read as a dictionary. - # Example of yaml structure: - # ======================================================================== - # versions: - # '2.25': - # '2.23': - # versionsuffix: '-R-4.0.0' - # ======================================================================== - if isinstance(versions, dict): - for version in versions: - check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) - if versions[version] is not None: - version_spec = versions[version] - if 'versionsuffix' in version_spec: - versionsuffix = str(version_spec['versionsuffix']) - else: - versionsuffix = '' - if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec): - lab_err = "EasyStack specifications of '%s' in %s " - lab_err += "contain labels. Labels aren't supported yet." - raise EasyBuildError(lab_err, name, filepath) - else: - versionsuffix = '' - - specs = { - 'name': name, - 'toolchain_name': toolchain_name, - 'toolchain_version': toolchain_version, - 'version': version, - 'versionsuffix': versionsuffix, - } - sw = SoftwareSpecs(**specs) - - # append newly created class instance to the list in instance of EasyStack class - easystack.software_list.append(sw) - continue - - elif isinstance(versions, (list, tuple)): - pass - - # multiple lines without ':' is read as a single string; example: - # versions: - # '2.24' - # '2.51' - elif isinstance(versions, string_type): - versions = versions.split() - - # single values like '2.24' should be wrapped in a list - else: - versions = [versions] - - # if version is not a dictionary, versionsuffix is not specified - versionsuffix = '' - - for version in versions: - check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) - sw = SoftwareSpecs( - name=name, version=version, versionsuffix=versionsuffix, - toolchain_name=toolchain_name, toolchain_version=toolchain_version) - # append newly created class instance to the list in instance of EasyStack class - easystack.software_list.append(sw) - - # assign general easystack attributes - easystack.easybuild_version = easybuild_version - easystack.robot = robot - - return easystack - @only_if_module_is_available('yaml', pkgname='PyYAML') def parse_easystack(filepath): @@ -321,18 +195,17 @@ def parse_easystack(filepath): # class instance which contains all info about planned build easystack = EasyStackParser.parse(filepath) - easyconfig_names = easystack.compose_ec_filenames() - # Disabled general options for now. We weren't using them, and first want support for EasyConfig-specific options. # Then, we need a method to resolve conflicts (specific options should win) # general_options = easystack.get_general_options() - _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names))) - _log.debug("Using EasyConfig specific options based on the following dict:") - _log.debug(easystack.ec_opts) + _log.debug("Parsed easystack:\n%s" % easystack) + +# _log.debug("Using EasyConfig specific options based on the following dict:") +# _log.debug(easystack.ec_opts) # if len(general_options) != 0: # _log.debug("General options for installation are: \n%s" % str(general_options)) # else: # _log.debug("No general options were specified in easystack") - return easyconfig_names, easystack.ec_opts + return easystack diff --git a/easybuild/main.py b/easybuild/main.py index d6d07ef2cf..4c357a3689 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -48,6 +48,7 @@ from easybuild.framework.easyblock import build_and_install_one, inject_checksums, inject_checksums_to_json from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig import easyconfig from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename @@ -67,7 +68,7 @@ from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import set_up_configuration, use_color +from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs @@ -214,143 +215,86 @@ def clean_exit(logfile, tmpdir, testing, silent=False): sys.exit(0) -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): +def process_easystack(easystack_path, args, logfile, testing, init_session_state, do_build): """ - Main function: parse command line options, and act accordingly. - :param args: command line arguments to use + Process an EasyStack file. That means, parsing, looping over all items in the EasyStack file + building (where requested) the individual items, etc + + :param easystack_path: path to EasyStack file to be processed + :param args: original command line arguments as received by main() :param logfile: log file to use - :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param init_session_state: initial session state, to use in test reports + :param do_build: whether or not to actually perform the build """ - - register_lock_cleanup_signal_handlers() - - # if $CDPATH is set, unset it, it'll only cause trouble... - # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] - - # When EB is run via `exec` the special bash variable $_ is not set - # So emulate this here to allow (module) scripts depending on that to work - if '_' not in os.environ: - os.environ['_'] = sys.executable - - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in - init_session_state = session_state() - - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - options, orig_paths = eb_go.options, eb_go.args + easystack = parse_easystack(easystack_path) global _log - (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, - from_pr_list, tweaked_ecs_paths) = cfg_settings - - # load hook implementations (if any) - hooks = load_hooks(options.hooks) - run_hook(START, hooks) - - if modtool is None: + # TODO: insert fast loop that validates if all command line options are valid. If there are errors in options, + # we want to know early on, and this loop potentially builds a lot of packages and could take very long + # for path in orig_paths: + # validate_command_opts(args, opts_per_ec[path]) + + # Loop over each item in the EasyStack file, each time updating the config + # This is because each item in an EasyStack file can have options associated with it + do_cleanup = True + for (path, ec_opts) in easystack.ec_opt_tuples: + _log.debug("Starting build for %s" % path) + # wipe easyconfig caches + easyconfig._easyconfigs_cache.clear() + easyconfig._easyconfig_files_cache.clear() + + # If EasyConfig specific arguments were supplied in EasyStack file + # merge arguments with original command line args + if ec_opts is not None: + _log.debug("EasyConfig specific options have been specified for " + "%s in the EasyStack file: %s", path, ec_opts) + if args is None: + args = sys.argv[1:] + ec_args = opts_dict_to_eb_opts(ec_opts) + # By appending ec_args to args, ec_args take priority + new_args = args + ec_args + _log.info("Argument list for %s after merging command line arguments with EasyConfig specific " + "options from the EasyStack file: %s", path, new_args) + else: + # If no EasyConfig specific arguments are defined, use original args. + # That way,set_up_configuration restores the original config + new_args = args + + # Reconfigure + eb_go, cfg_settings = set_up_configuration(args=new_args, logfile=logfile, testing=testing, + reconfigure=True, silent=True) + # Since we reconfigure, we should also reload hooks and get current module tools + hooks = load_hooks(eb_go.options.hooks) modtool = modules_tool(testing=testing) - # check whether any (EasyBuild-generated) modules are loaded already in the current session - modtool.check_loaded_modules() - - if options.last_log: - # print location to last log file, and exit - last_log = find_last_log(logfile) or '(none)' - print_msg(last_log, log=_log, prefix=False) - - # if easystack is provided with the command, commands with arguments from it will be executed - if options.easystack: - # TODO add general_options (i.e. robot) to build options - orig_paths, general_options = parse_easystack(options.easystack) - if general_options: - print_warning("Specifying options in easystack files is not supported yet. They are parsed, but ignored.") - - # check whether packaging is supported when it's being used - if options.package: - check_pkg_support() - else: - _log.debug("Packaging not enabled, so not checking for packaging support.") - - # search for easyconfigs, if a query is specified - if search_query: - search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, - terse=options.terse) + # Process actual item in the EasyStack file + do_cleanup &= process_eb_args([path], eb_go, cfg_settings, modtool, testing, init_session_state, + hooks, do_build) - if options.check_eb_deps: - print_checks(check_easybuild_deps(modtool)) - - # GitHub options that warrant a silent cleanup & exit - if options.check_github: - check_github() + return do_cleanup - elif options.install_github_token: - install_github_token(options.github_user, silent=build_option('silent')) - - elif options.close_pr: - close_pr(options.close_pr, motivation_msg=options.close_pr_msg) - - elif options.list_prs: - print(list_prs(options.list_prs)) - elif options.merge_pr: - merge_pr(options.merge_pr) - - elif options.review_pr: - print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing, - max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter)) - - elif options.add_pr_labels: - add_pr_labels(options.add_pr_labels) - - elif options.list_installed_software: - detailed = options.list_installed_software == 'detailed' - print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) - - elif options.list_software: - print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) - - elif options.create_index: - print_msg("Creating index for %s..." % options.create_index, prefix=False) - index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age) - index = load_index(options.create_index) - print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) - - # non-verbose cleanup after handling GitHub integration stuff or printing terse info - early_stop_options = [ - options.add_pr_labels, - options.check_eb_deps, - options.check_github, - options.create_index, - options.install_github_token, - options.list_installed_software, - options.list_software, - options.close_pr, - options.list_prs, - options.merge_pr, - options.review_pr, - options.terse, - search_query, - ] - if any(early_stop_options): - clean_exit(logfile, eb_tmpdir, testing, silent=True) - - # update session state - eb_config = eb_go.generate_cmd_line(add_default=True) - modlist = modtool.list() # build options must be initialized first before 'module list' works - init_session_state.update({'easybuild_configuration': eb_config}) - init_session_state.update({'module_list': modlist}) - _log.debug("Initial session state: %s" % init_session_state) +def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session_state, hooks, do_build): + """ + Remainder of main function, actually process provided arguments (list of files/paths), + according to specified options. + + :param eb_args: list of arguments that were specified to 'eb' command (or an easystack file); + includes filenames/paths of files to process + (mostly easyconfig files, but can also includes patch files, etc.) + :param eb_go: EasyBuildOptions instance (option parser) + :param cfg_settings: as returned by set_up_configuration + :param modtool: the modules tool, as returned by modules_tool() + :param testing: bool whether we're running in test mode + :param init_session_state: initial session state, to use in test reports + :param hooks: hooks, as loaded by load_hooks from the options + :param do_build: whether or not to actually perform the build + """ + options = eb_go.options - if options.skip_test_step: - if options.ignore_test_failure: - raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. " - "Please use only one of them.") - else: - print_warning("Will not run the test step as requested via skip-test-step. " - "Consider using ignore-test-failure instead and verify the results afterwards") + global _log # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) @@ -358,18 +302,22 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: - if orig_paths: + if eb_args: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() - orig_paths.append(eb_file) + eb_args.append(eb_file) + + # Unpack cfg_settings + (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, + from_pr_list, tweaked_ecs_paths) = cfg_settings if options.copy_ec: # figure out list of files to copy + target location (taking into account --from-pr) - orig_paths, target_path = det_copy_ec_specs(orig_paths, from_pr_list) + eb_args, target_path = det_copy_ec_specs(eb_args, from_pr_list) - categorized_paths = categorize_files_by_type(orig_paths) + categorized_paths = categorize_files_by_type(eb_args) # command line options that do not require any easyconfigs to be specified pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr @@ -388,7 +336,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.copy_ec: # at this point some paths may still just be filenames rather than absolute paths, # so try to determine full path for those too via robot search path - paths = locate_files(orig_paths, robot_path) + paths = locate_files(eb_args, robot_path) copy_files(paths, target_path, target_single_file=True, allow_empty=False, verbose=True) @@ -400,7 +348,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print_msg("Contents of %s:" % path) print_msg(read_file(path), prefix=False) - clean_exit(logfile, eb_tmpdir, testing) + return True if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated @@ -431,13 +379,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): - clean_exit(logfile, eb_tmpdir, testing) + return True # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") - for easyconfig in easyconfigs: - verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) + for ec in easyconfigs: + verify_easyconfig_filename(ec['spec'], ec['ec'], parsed_ec=ec['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail @@ -448,7 +396,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) - clean_exit(logfile, eb_tmpdir, testing) + return True forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules @@ -467,8 +415,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # keep track for which easyconfigs we should set the corresponding module as default if options.set_default_module: - for easyconfig in easyconfigs: - easyconfig['ec'].set_default_module = True + for ec in easyconfigs: + ec['ec'].set_default_module = True # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: @@ -495,7 +443,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True) - clean_exit(logfile, eb_tmpdir, testing) + return True # creating/updating PRs if pr_options: @@ -542,26 +490,32 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): inject_checksums(ordered_ecs, options.inject_checksums) elif options.inject_checksums_to_json: - inject_checksums_to_json(ordered_ecs, options.inject_checksums_to_json) + with rich_live_cm(): + inject_checksums_to_json(ordered_ecs, options.inject_checksums_to_json) # cleanup and exit after dry run, searching easyconfigs or submitting regression test - stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] - stop_options += [options.inject_checksums_to_json] + stop_options = [ + dry_run_mode, + options.check_conflicts, + options.dump_env_script, + options.inject_checksums, + options.inject_checksums_to_json, + ] if any(no_ec_opts) or any(stop_options): - clean_exit(logfile, eb_tmpdir, testing) + return True # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) - clean_exit(logfile, eb_tmpdir, testing, silent=True) + return True # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") - clean_exit(logfile, eb_tmpdir, testing) + return True # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): @@ -597,10 +551,156 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): run_hook(END, hooks) + return overall_success + + +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): + """ + Main function: parse command line options, and act accordingly. + :param args: command line arguments to use + :param logfile: log file to use + :param do_build: whether or not to actually perform the build + :param testing: enable testing mode + """ + + register_lock_cleanup_signal_handlers() + + # if $CDPATH is set, unset it, it'll only cause trouble... + # see https://github.com/easybuilders/easybuild-framework/issues/2944 + if 'CDPATH' in os.environ: + del os.environ['CDPATH'] + + # When EB is run via `exec` the special bash variable $_ is not set + # So emulate this here to allow (module) scripts depending on that to work + if '_' not in os.environ: + os.environ['_'] = sys.executable + + # purposely session state very early, to avoid modules loaded by EasyBuild meddling in + init_session_state = session_state() + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + options, orig_paths = eb_go.options, eb_go.args + + global _log + (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, + from_pr_list, tweaked_ecs_paths) = cfg_settings + + # load hook implementations (if any) + hooks = load_hooks(options.hooks) + + run_hook(START, hooks) + + if modtool is None: + modtool = modules_tool(testing=testing) + + # check whether any (EasyBuild-generated) modules are loaded already in the current session + modtool.check_loaded_modules() + + if options.last_log: + # print location to last log file, and exit + last_log = find_last_log(logfile) or '(none)' + print_msg(last_log, log=_log, prefix=False) + + # check whether packaging is supported when it's being used + if options.package: + check_pkg_support() + else: + _log.debug("Packaging not enabled, so not checking for packaging support.") + + # search for easyconfigs, if a query is specified + if search_query: + search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, + terse=options.terse) + + if options.check_eb_deps: + print_checks(check_easybuild_deps(modtool)) + + # GitHub options that warrant a silent cleanup & exit + if options.check_github: + check_github() + + elif options.install_github_token: + install_github_token(options.github_user, silent=build_option('silent')) + + elif options.close_pr: + close_pr(options.close_pr, motivation_msg=options.close_pr_msg) + + elif options.list_prs: + print(list_prs(options.list_prs)) + + elif options.merge_pr: + merge_pr(options.merge_pr) + + elif options.review_pr: + print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing, + max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter)) + + elif options.add_pr_labels: + add_pr_labels(options.add_pr_labels) + + elif options.list_installed_software: + detailed = options.list_installed_software == 'detailed' + print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) + + elif options.list_software: + print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) + + elif options.create_index: + print_msg("Creating index for %s..." % options.create_index, prefix=False) + index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age) + index = load_index(options.create_index) + print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) + + # non-verbose cleanup after handling GitHub integration stuff or printing terse info + early_stop_options = [ + options.add_pr_labels, + options.check_eb_deps, + options.check_github, + options.create_index, + options.install_github_token, + options.list_installed_software, + options.list_software, + options.close_pr, + options.list_prs, + options.merge_pr, + options.review_pr, + options.terse, + search_query, + ] + if any(early_stop_options): + clean_exit(logfile, eb_tmpdir, testing, silent=True) + + # update session state + eb_config = eb_go.generate_cmd_line(add_default=True) + modlist = modtool.list() # build options must be initialized first before 'module list' works + init_session_state.update({'easybuild_configuration': eb_config}) + init_session_state.update({'module_list': modlist}) + _log.debug("Initial session state: %s" % init_session_state) + + if options.skip_test_step: + if options.ignore_test_failure: + raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. " + "Please use only one of them.") + else: + print_warning("Will not run the test step as requested via skip-test-step. " + "Consider using ignore-test-failure instead and verify the results afterwards") + + # if EasyStack file is provided, parse it, and loop over the items in the EasyStack file + if options.easystack: + if len(orig_paths) > 0: + msg = '\n'.join([ + "Passing additional arguments when building from an EasyStack file is not supported.", + "The following arguments will be ignored:", + ] + orig_paths) + print_warning(msg) + do_cleanup = process_easystack(options.easystack, args, logfile, testing, init_session_state, do_build) + else: + do_cleanup = process_eb_args(orig_paths, eb_go, cfg_settings, modtool, testing, init_session_state, + hooks, do_build) + # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) - if overall_success: - cleanup(logfile, eb_tmpdir, testing) + if do_cleanup: + cleanup(logfile, eb_tmpdir, testing, silent=False) if __name__ == "__main__": diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3220f4def2..c92ca97e20 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -74,6 +74,7 @@ from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path +from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses @@ -1503,7 +1504,7 @@ def check_root_usage(allow_use_as_root=False): "so let's end this here.") -def set_up_configuration(args=None, logfile=None, testing=False, silent=False): +def set_up_configuration(args=None, logfile=None, testing=False, silent=False, reconfigure=False): """ Set up EasyBuild configuration, by parsing configuration settings & initialising build options. @@ -1511,6 +1512,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): :param logfile: log file to use :param testing: enable testing mode :param silent: stay silent (no printing) + :param reconfigure: reconfigure singletons that hold configuration dictionaries. Use with care: normally, + configuration shouldn't be changed during a run. Exceptions are when looping over items in EasyStack files """ # set up fake 'vsc' Python package, to catch easyblocks/scripts that still import from vsc.* namespace @@ -1588,6 +1591,21 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } + + # Remove existing singletons if reconfigure==True (allows reconfiguration when looping over EasyStack items) + if reconfigure: + BuildOptions.__class__._instances.clear() + ConfigurationVariables.__class__._instances.clear() + elif len(BuildOptions.__class__._instances) + len(ConfigurationVariables.__class__._instances) > 0: + msg = '\n'.join([ + "set_up_configuration is about to call init() and init_build_options().", + "However, the singletons that these functions normally initialize already exist.", + "If configuration should be changed, this may lead to unexpected behavior," + "as the existing singletons will be returned. If you intended to reconfigure", + "you should probably pass reconfigure=True to set_up_configuration()." + ]) + print_warning(msg, log=log) + # initialise the EasyBuild configuration & build options init(options, config_options_dict) init_build_options(build_options=build_options, cmdline_options=options) @@ -1885,3 +1903,35 @@ def set_tmpdir(tmpdir=None, raise_error=False): raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir + + +def opts_dict_to_eb_opts(args_dict): + """ + Convert a dictionary with configuration option values to command-line options for the 'eb' command. + Can by used to convert e.g. easyconfig-specific options from an easystack file to a list of strings + that can be fed into the EasyBuild option parser + :param args_dict: dictionary with configuration option values + :return: a list of strings representing command-line options for the 'eb' command + """ + + _log.debug("Converting dictionary %s to argument list" % args_dict) + args = [] + for arg in sorted(args_dict): + if len(arg) == 1: + prefix = '-' + else: + prefix = '--' + option = prefix + str(arg) + value = args_dict[arg] + if isinstance(value, (list, tuple)): + value = ','.join(str(x) for x in value) + + if value in [True, None]: + args.append(option) + elif value is False: + args.append('--disable-' + option[2:]) + elif value is not None: + args.append(option + '=' + str(value)) + + _log.debug("Converted dictionary %s to argument list %s" % (args_dict, args)) + return args diff --git a/test/framework/easystack.py b/test/framework/easystack.py index a0e2595531..f9b971428d 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -57,104 +57,90 @@ def tearDown(self): super(EasyStackTest, self).tearDown() def test_easystack_basic(self): - """Test for basic easystack file.""" + """Test for basic easystack files.""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') - ec_fns, opts = parse_easystack(test_easystack) - expected = [ - 'binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb', - 'foss-2018a.eb', - 'toy-0.0-gompi-2018a-test.eb', + test_easystacks = [ + 'test_easystack_basic.yaml', + 'test_easystack_basic_dict.yaml', + 'test_easystack_easyconfigs_with_eb_ext.yaml', ] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) - - def test_easystack_easyconfigs(self): - """Test for easystack file using 'easyconfigs' key.""" + for fn in test_easystacks: + test_easystack = os.path.join(topdir, 'easystacks', fn) + + easystack = parse_easystack(test_easystack) + expected = [ + 'binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb', + 'foss-2018a.eb', + 'toy-0.0-gompi-2018a-test.eb', + ] + self.assertEqual(sorted([x[0] for x in easystack.ec_opt_tuples]), sorted(expected)) + self.assertTrue(all(x[1] is None for x in easystack.ec_opt_tuples)) + + def test_easystack_easyconfigs_dict(self): + """Test for easystack file where easyconfigs item is parsed as a dict, because easyconfig names are not + prefixed by dashes""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs.yaml') + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_dict.yaml') - ec_fns, opts = parse_easystack(test_easystack) - expected = [ - 'binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb', - 'foss-2018a.eb', - 'toy-0.0-gompi-2018a-test.eb', - ] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) + error_pattern = r"Found dict value for 'easyconfigs' in .* should be list.\nMake sure you use '-' to create .*" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - def test_easystack_easyconfigs_with_eb_ext(self): - """Test for easystack file using 'easyconfigs' key, where eb extension is included in the easystack file""" + def test_easystack_easyconfigs_str(self): + """Test for easystack file where easyconfigs item is parsed as a dict, because easyconfig names are not + prefixed by dashes""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_with_eb_ext.yaml') + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_str.yaml') - ec_fns, opts = parse_easystack(test_easystack) - expected = [ - 'binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb', - 'foss-2018a.eb', - 'toy-0.0-gompi-2018a-test.eb', - ] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) + error_pattern = r"Found str value for 'easyconfigs' in .* should be list.\nMake sure you use '-' to create .*" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_easyconfig_opts(self): """Test an easystack file using the 'easyconfigs' key, with additonal options for some easyconfigs""" topdir = os.path.dirname(os.path.abspath(__file__)) test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_opts.yaml') - ec_fns, opts = parse_easystack(test_easystack) - expected = [ - 'binutils-2.25-GCCcore-4.9.3.eb', - 'binutils-2.26-GCCcore-4.9.3.eb', - 'foss-2018a.eb', - 'toy-0.0-gompi-2018a-test.eb', + easystack = parse_easystack(test_easystack) + expected_tuples = [ + ('binutils-2.25-GCCcore-4.9.3.eb', {'debug': True, 'from-pr': 12345}), + ('binutils-2.26-GCCcore-4.9.3.eb', None), + ('foss-2018a.eb', {'enforce-checksums': True, 'robot': True}), + ('toy-0.0-gompi-2018a-test.eb', None), ] - expected_opts = { - 'binutils-2.25-GCCcore-4.9.3.eb': {'debug': True, 'from-pr': 12345}, - 'foss-2018a.eb': {'enforce-checksums': True, 'robot': True}, - } - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, expected_opts) - - def test_parse_fail(self): - """Test for clean error when easystack file fails to parse.""" - test_yml = os.path.join(self.test_prefix, 'test.yml') - write_file(test_yml, 'software: %s') - error_pattern = "Failed to parse .*/test.yml: while scanning for the next token" - self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_yml) + self.assertEqual(easystack.ec_opt_tuples, expected_tuples) - def test_easystack_wrong_structure(self): - """Test for --easystack when yaml easystack has wrong structure""" + def test_easystack_invalid_key(self): + """Test easystack files with invalid key at the same level as the 'options' key""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_invalid_key.yaml') - expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" - expected_err += r"( 'float' object is not subscriptable[\S\s]*" - expected_err += r"| 'float' object is unsubscriptable" - expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) + error_pattern = r"Found one or more invalid keys for .* \(only 'options' supported\).*" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - def test_easystack_asterisk(self): - """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + def test_easystack_invalid_key2(self): + """Test easystack files with invalid key at the same level as the key that names the easyconfig""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_invalid_key2.yaml') - expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " - expected_err += "Wildcard feature is not supported yet." + error_pattern = r"expected a dictionary with one key \(the EasyConfig name\), " + error_pattern += r"instead found keys: .*, invalid_key" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) - - def test_easystack_labels(self): + def test_missing_easyconfigs_key(self): + """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" topdir = os.path.dirname(os.path.abspath(__file__)) - test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + test_easystack = os.path.join(topdir, 'easystacks', 'test_missing_easyconfigs_key.yaml') + + error_pattern = r"Top-level key 'easyconfigs' missing in easystack file %s" % test_easystack + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " - error_msg += "Labels aren't supported yet." - self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, test_easystack) + def test_parse_fail(self): + """Test for clean error when easystack file fails to parse.""" + test_yml = os.path.join(self.test_prefix, 'test.yml') + write_file(test_yml, 'easyconfigs: %s') + error_pattern = "Failed to parse .*/test.yml: while scanning for the next token" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_yml) def test_check_value(self): """Test check_value function.""" @@ -168,104 +154,6 @@ def test_check_value(self): error_pattern = r"Value .* \(of type .*\) obtained for is not valid!" self.assertErrorRegex(EasyBuildError, error_pattern, check_value, version, context) - def test_easystack_versions(self): - """Test handling of versions in easystack files.""" - - test_easystack = os.path.join(self.test_prefix, 'test.yml') - tmpl_easystack_txt = '\n'.join([ - "software:", - " foo:", - " toolchains:", - " SYSTEM:", - " versions:", - ]) - - # normal versions, which are not treated special by YAML: no single quotes needed - versions = ('1.2.3', '1.2.30', '2021a', '1.2.3') - for version in versions: - write_file(test_easystack, tmpl_easystack_txt + ' ' + version) - ec_fns, opts = parse_easystack(test_easystack) - self.assertEqual(ec_fns, ['foo-%s.eb' % version]) - self.assertEqual(opts, {}) - - # multiple versions as a list - test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]" - write_file(test_easystack, test_easystack_txt) - ec_fns, opts = parse_easystack(test_easystack) - expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb'] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) - - # multiple versions listed with more info - test_easystack_txt = '\n'.join([ - tmpl_easystack_txt, - " 1.2.3:", - " 2021a:", - " 3.2.1:", - " versionsuffix: -foo", - ]) - write_file(test_easystack, test_easystack_txt) - ec_fns, opts = parse_easystack(test_easystack) - expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb'] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) - - # versions that get interpreted by YAML as float or int, single quotes required - for version in ('1.2', '123', '3.50', '100', '2.44_01'): - error_pattern = r"Value .* \(of type .*\) obtained for foo \(with system toolchain\) is not valid\!" - - write_file(test_easystack, tmpl_easystack_txt + ' ' + version) - self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - - # all is fine when wrapping the value in single quotes - write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'") - ec_fns, opts = parse_easystack(test_easystack) - self.assertEqual(ec_fns, ['foo-%s.eb' % version]) - self.assertEqual(opts, {}) - - # one rotten apple in the basket is enough - test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version - write_file(test_easystack, test_easystack_txt) - self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - - test_easystack_txt = '\n'.join([ - tmpl_easystack_txt, - " 1.2.3:", - " %s:" % version, - " 3.2.1:", - " versionsuffix: -foo", - ]) - write_file(test_easystack, test_easystack_txt) - self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) - - # single quotes to the rescue! - test_easystack_txt = '\n'.join([ - tmpl_easystack_txt, - " 1.2.3:", - " '%s':" % version, - " 3.2.1:", - " versionsuffix: -foo", - ]) - write_file(test_easystack, test_easystack_txt) - ec_fns, opts = parse_easystack(test_easystack) - expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb'] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) - - # also check toolchain version that could be interpreted as a non-string value... - test_easystack_txt = '\n'.join([ - 'software:', - ' test:', - ' toolchains:', - ' intel-2021.03:', - " versions: [1.2.3, '2.3']", - ]) - write_file(test_easystack, test_easystack_txt) - ec_fns, opts = parse_easystack(test_easystack) - expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb'] - self.assertEqual(sorted(ec_fns), sorted(expected)) - self.assertEqual(opts, {}) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easystacks/test_easystack_asterisk.yaml b/test/framework/easystacks/test_easystack_asterisk.yaml deleted file mode 100644 index 7f440636cd..0000000000 --- a/test/framework/easystacks/test_easystack_asterisk.yaml +++ /dev/null @@ -1,6 +0,0 @@ -software: - binutils: - toolchains: - GCCcore-4.9.3: - versions: - "2.11.*" \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml index 491f113f4a..1d7bfe073d 100644 --- a/test/framework/easystacks/test_easystack_basic.yaml +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -1,17 +1,5 @@ -software: - binutils: - toolchains: - GCCcore-4.9.3: - versions: - '2.25': - '2.26': - foss: - toolchains: - SYSTEM: - versions: [2018a] - toy: - toolchains: - gompi-2018a: - versions: - '0.0': - versionsuffix: '-test' +easyconfigs: + - binutils-2.25-GCCcore-4.9.3 + - binutils-2.26-GCCcore-4.9.3 + - foss-2018a + - toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_basic_dict.yaml b/test/framework/easystacks/test_easystack_basic_dict.yaml new file mode 100644 index 0000000000..6a1f839a11 --- /dev/null +++ b/test/framework/easystacks/test_easystack_basic_dict.yaml @@ -0,0 +1,5 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3: + - binutils-2.26-GCCcore-4.9.3: + - foss-2018a: + - toy-0.0-gompi-2018a-test: diff --git a/test/framework/easystacks/test_easystack_easyconfigs.yaml b/test/framework/easystacks/test_easystack_easyconfigs.yaml deleted file mode 100644 index 1d7bfe073d..0000000000 --- a/test/framework/easystacks/test_easystack_easyconfigs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - binutils-2.25-GCCcore-4.9.3 - - binutils-2.26-GCCcore-4.9.3 - - foss-2018a - - toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_easyconfigs_dict.yaml b/test/framework/easystacks/test_easystack_easyconfigs_dict.yaml new file mode 100644 index 0000000000..a6f7d96d78 --- /dev/null +++ b/test/framework/easystacks/test_easystack_easyconfigs_dict.yaml @@ -0,0 +1,5 @@ +easyconfigs: + binutils-2.25-GCCcore-4.9.3: + binutils-2.26-GCCcore-4.9.3: + foss-2018a: + toy-0.0-gompi-2018a-test: diff --git a/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml b/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml index d2bc703986..c8686466a1 100644 --- a/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml +++ b/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml @@ -6,8 +6,9 @@ easyconfigs: } - binutils-2.26-GCCcore-4.9.3 - foss-2018a: - options: { - 'enforce-checksums': True, - 'robot': True, - } + # no strict need for quotes as long as YAML parser doesn't get smart with value type (booleans, integers, ...); + # { and } are also optional in YAML to create a dictionary + options: + enforce-checksums: True + robot: True - toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_easyconfigs_str.yaml b/test/framework/easystacks/test_easystack_easyconfigs_str.yaml new file mode 100644 index 0000000000..ddfefd0b47 --- /dev/null +++ b/test/framework/easystacks/test_easystack_easyconfigs_str.yaml @@ -0,0 +1,5 @@ +easyconfigs: + binutils-2.25-GCCcore-4.9.3 + binutils-2.26-GCCcore-4.9.3 + foss-2018a + toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_invalid_key.yaml b/test/framework/easystacks/test_easystack_invalid_key.yaml new file mode 100644 index 0000000000..ad4dfd3d71 --- /dev/null +++ b/test/framework/easystacks/test_easystack_invalid_key.yaml @@ -0,0 +1,14 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3: + options: { + 'debug': True, + 'from-pr': 12345, + } + invalid_key: {} + - binutils-2.26-GCCcore-4.9.3 + - foss-2018a: + options: { + 'enforce-checksums': True, + 'robot': True, + } + - toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_invalid_key2.yaml b/test/framework/easystacks/test_easystack_invalid_key2.yaml new file mode 100644 index 0000000000..5016194248 --- /dev/null +++ b/test/framework/easystacks/test_easystack_invalid_key2.yaml @@ -0,0 +1,14 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3: + options: { + 'debug': True, + 'from-pr': 12345, + } + invalid_key: {} + - binutils-2.26-GCCcore-4.9.3 + - foss-2018a: + options: { + 'enforce-checksums': True, + 'robot': True, + } + - toy-0.0-gompi-2018a-test diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml deleted file mode 100644 index f00db0e249..0000000000 --- a/test/framework/easystacks/test_easystack_labels.yaml +++ /dev/null @@ -1,7 +0,0 @@ -software: - binutils: - toolchains: - GCCcore-4.9.3: - versions: - '3.11': - exclude-labels: arch:aarch64 diff --git a/test/framework/easystacks/test_easystack_wrong_structure.yaml b/test/framework/easystacks/test_easystack_wrong_structure.yaml deleted file mode 100644 index a328b5413b..0000000000 --- a/test/framework/easystacks/test_easystack_wrong_structure.yaml +++ /dev/null @@ -1,6 +0,0 @@ -software: - Bioconductor: - toolchains: - # foss-2020a: - versions: - 3.11 \ No newline at end of file diff --git a/test/framework/easystacks/test_missing_easyconfigs_key.yaml b/test/framework/easystacks/test_missing_easyconfigs_key.yaml new file mode 100644 index 0000000000..d8be11fdfa --- /dev/null +++ b/test/framework/easystacks/test_missing_easyconfigs_key.yaml @@ -0,0 +1,4 @@ +- binutils-2.25-GCCcore-4.9.3.eb +- binutils-2.26-GCCcore-4.9.3.eb +- foss-2018a.eb +- toy-0.0-gompi-2018a-test.eb diff --git a/test/framework/options.py b/test/framework/options.py index a64bd2749b..a4ac53095a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -49,23 +49,25 @@ from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import DEFAULT_MODULECLASSES -from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes +from easybuild.tools.build_log import EasyBuildError, EasyBuildLog +from easybuild.tools.config import DEFAULT_MODULECLASSES, BuildOptions, ConfigurationVariables +from easybuild.tools.config import build_option, find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, download_file from easybuild.tools.filetools import is_patch_file, mkdir, move_file, parse_http_header_fields_urlpat from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token +from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.options import EasyBuildOptions, parse_external_modules_metadata, set_tmpdir, use_color +from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata +from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config try: import pycodestyle # noqa @@ -438,7 +440,7 @@ def check_args(job_args, passed_args=None): eb_file, '--job', ] + job_args - outtxt = self.eb_main(args) + outtxt = self.eb_main(args, raise_error=True) job_msg = r"INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(passed_args) assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) @@ -6563,10 +6565,12 @@ def test_easystack_basic(self): args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run'] stdout = self.eb_main(args, do_build=True, raise_error=True) patterns = [ - r"[\S\s]*INFO Building from easystack:[\S\s]*", - r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs: " - r"binutils-2.25-GCCcore-4.9.3.eb, binutils-2.26-GCCcore-4.9.3.eb, " - r"foss-2018a.eb, toy-0.0-gompi-2018a-test.eb", + r"INFO Building from easystack:", + r"DEBUG Parsed easystack:\n" + ".*binutils-2.25-GCCcore-4.9.3.eb.*\n" + ".*binutils-2.26-GCCcore-4.9.3.eb.*\n" + ".*foss-2018a.eb.*\n" + ".*toy-0.0-gompi-2018a-test.eb.*", r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb \(module: binutils/2.25-GCCcore-4.9.3\)", r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb \(module: binutils/2.26-GCCcore-4.9.3\)", r"\* \[ \] .*/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb \(module: toy/0.0-gompi-2018a-test\)", @@ -6576,6 +6580,216 @@ def test_easystack_basic(self): regex = re.compile(pattern) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_easystack_opts(self): + """Test for easystack file that specifies options for specific easyconfigs.""" + + robot_paths = os.environ['EASYBUILD_ROBOT_PATHS'] + hidden_installpath = os.path.join(self.test_installpath, 'hidden') + + test_es_txt = '\n'.join([ + "easyconfigs:", + " - toy-0.0:", + " options:", + " force: True", + " hidden: True", + " installpath: %s" % hidden_installpath, + " - libtoy-0.0:", + " options:", + " force: True", + " robot: ~", + " robot-paths: %s:%s" % (robot_paths, self.test_prefix), + ]) + test_es_path = os.path.join(self.test_prefix, 'test.yml') + write_file(test_es_path, test_es_txt) + + mod_dir = os.path.join(self.test_installpath, 'modules', 'all') + + # touch module file for libtoy, so we can check whether the existing module is replaced + libtoy_mod = os.path.join(mod_dir, 'libtoy', '0.0') + write_file(libtoy_mod, ModuleGeneratorTcl.MODULE_SHEBANG) + + del os.environ['EASYBUILD_INSTALLPATH'] + args = [ + '--experimental', + '--easystack', test_es_path, + '--installpath', self.test_installpath, + ] + self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False) + + mod_ext = '.lua' if get_module_syntax() == 'Lua' else '' + + # make sure that $EBROOTLIBTOY is not defined + if 'EBROOTLIBTOY' in os.environ: + del os.environ['EBROOTLIBTOY'] + + # libtoy module should be installed, module file should at least set EBROOTLIBTOY + mod_dir = os.path.join(self.test_installpath, 'modules', 'all') + mod_path = os.path.join(mod_dir, 'libtoy', '0.0') + mod_ext + self.assertTrue(os.path.exists(mod_path)) + self.modtool.use(mod_dir) + self.modtool.load(['libtoy']) + self.assertTrue(os.path.exists(os.environ['EBROOTLIBTOY'])) + + # module should be hidden and in different install path + mod_path = os.path.join(hidden_installpath, 'modules', 'all', 'toy', '.0.0') + mod_ext + self.assertTrue(os.path.exists(mod_path)) + + # check build options that were put in place for last easyconfig + self.assertFalse(build_option('hidden')) + self.assertTrue(build_option('force')) + self.assertEqual(build_option('robot'), [robot_paths, self.test_prefix]) + + def test_easystack_easyconfigs_cache(self): + """ + Test for easystack file that specifies same easyconfig twice, + but from a different location. + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + libtoy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'l', 'libtoy', 'libtoy-0.0.eb') + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\ndependencies = [('libtoy', '0.0')]" + write_file(test_ec, test_ec_txt) + + test_subdir = os.path.join(self.test_prefix, 'deps') + mkdir(test_subdir, parents=True) + copy_file(libtoy_ec, test_subdir) + + test_es_txt = '\n'.join([ + "easyconfigs:", + " - toy-0.0", + " - toy-0.0:", + " options:", + " robot: %s:%s" % (test_subdir, self.test_prefix), + ]) + test_es_path = os.path.join(self.test_prefix, 'test.yml') + write_file(test_es_path, test_es_txt) + + args = [ + '--experimental', + '--easystack', test_es_path, + '--dry-run', + '--robot=%s' % self.test_prefix, + ] + stdout = self.eb_main(args, do_build=True, raise_error=True, redo_init_config=False) + + # check whether libtoy-0.0.eb comes from 2nd + regex = re.compile(r"^ \* \[ \] %s" % libtoy_ec, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + regex = re.compile(r"^ \* \[ \] %s" % os.path.join(test_subdir, 'libtoy-0.0.eb'), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + def test_set_up_configuration(self): + """Tests for set_up_configuration function.""" + + # check default configuration first + self.assertFalse(build_option('debug')) + self.assertFalse(build_option('hidden')) + # tests may be configured to run with Tcl module syntax + self.assertTrue(get_module_syntax() in ('Lua', 'Tcl')) + + # start with a clean slate, reset all configuration done by setUp method that prepares each test + cleanup() + + os.environ['EASYBUILD_PREFIX'] = self.test_prefix + eb_go, settings = set_up_configuration(args=['--debug', '--module-syntax=Tcl'], silent=True) + + # 2nd part of return value is a tuple with various settings + self.assertTrue(isinstance(settings, tuple)) + self.assertEqual(len(settings), 9) + self.assertEqual(settings[0], {}) # build specs + self.assertTrue(isinstance(settings[1], EasyBuildLog)) # EasyBuildLog instance + self.assertTrue(settings[2].endswith('.log')) # path to log file + self.assertTrue(os.path.exists(settings[2])) + self.assertTrue(isinstance(settings[3], list)) # list of robot paths + self.assertEqual(len(settings[3]), 1) + self.assertTrue(os.path.samefile(settings[3][0], os.environ['EASYBUILD_ROBOT_PATHS'])) + self.assertEqual(settings[4], None) # search query + self.assertTrue(os.path.samefile(settings[5], tempfile.gettempdir())) # tmpdir + self.assertEqual(settings[6], False) # try_to_generate + self.assertEqual(settings[7], []) # from_prs list + self.assertEqual(settings[8], None) # list of paths for tweaked ecs + + self.assertEqual(eb_go.options.prefix, self.test_prefix) + self.assertTrue(eb_go.options.debug) + self.assertEqual(eb_go.options.module_syntax, 'Tcl') + + # set_up_configuration also initializes build options and configuration variables (both Singleton classes) + self.assertTrue(build_option('debug')) + self.assertTrue(BuildOptions()['debug']) + + self.assertEqual(ConfigurationVariables()['module_syntax'], 'Tcl') + self.assertEqual(get_module_syntax(), 'Tcl') + + self.assertFalse(BuildOptions()['hidden']) + self.assertFalse(build_option('hidden')) + + # calling set_up_configuration again triggers a warning being printed, + # because build options and configuration variables will not be re-configured by default! + self.mock_stderr(True) + eb_go, _ = set_up_configuration(args=['--hidden'], silent=True) + stderr = self.get_stderr() + self.mock_stderr(False) + + self.assertTrue("WARNING: set_up_configuration is about to call init() and init_build_options()" in stderr) + + # 'hidden' option is enabled, but corresponding build option is still set to False! + self.assertTrue(eb_go.options.hidden) + self.assertFalse(BuildOptions()['hidden']) + self.assertFalse(build_option('hidden')) + + self.assertEqual(eb_go.options.prefix, self.test_prefix) + + self.assertTrue(build_option('debug')) + self.assertTrue(BuildOptions()['debug']) + + self.assertEqual(ConfigurationVariables()['module_syntax'], 'Tcl') + self.assertEqual(get_module_syntax(), 'Tcl') + + # build options and configuration variables are only re-initialized on demand + eb_go, _ = set_up_configuration(args=['--hidden'], silent=True, reconfigure=True) + + self.assertTrue(eb_go.options.hidden) + self.assertTrue(BuildOptions()['hidden']) + self.assertTrue(build_option('hidden')) + + self.assertEqual(eb_go.options.prefix, self.test_prefix) + + self.assertFalse(build_option('debug')) + self.assertFalse(BuildOptions()['debug']) + + # tests may be configured to run with Tcl module syntax + self.assertTrue(ConfigurationVariables()['module_syntax'] in ('Lua', 'Tcl')) + self.assertTrue(get_module_syntax() in ('Lua', 'Tcl')) + + def test_opts_dict_to_eb_opts(self): + """Tests for opts_dict_to_eb_opts.""" + + self.assertEqual(opts_dict_to_eb_opts({}), []) + self.assertEqual(opts_dict_to_eb_opts({'foo': '123'}), ['--foo=123']) + + opts_dict = { + 'module-syntax': 'Tcl', + # multi-value option + 'from-pr': [1234, 2345], + # enabled boolean options + 'robot': None, + 'force': True, + # disabled boolean option + 'debug': False, + } + expected = [ + '--disable-debug', + '--force', + '--from-pr=1234,2345', + '--module-syntax=Tcl', + '--robot', + ] + self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected) + def suite(): """ returns all the testcases in this module """