From c8551d036ce178aeed450f681f8f864965e73565 Mon Sep 17 00:00:00 2001 From: Erich L Foster Date: Thu, 20 Apr 2017 09:27:09 -0600 Subject: [PATCH 1/5] Renamed manage_case to query_case Renamed manage_case to query_case as per #1155. Additionally, changed all references to manage_case to query_case. --- scripts/create_newcase | 10 +++++----- scripts/create_test | 2 +- scripts/lib/CIME/XML/grids.py | 2 +- scripts/{manage_case => query_case} | 0 tools/Readme.configure | 4 ++-- tools/configure | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) rename scripts/{manage_case => query_case} (100%) diff --git a/scripts/create_newcase b/scripts/create_newcase index 627c0b60f42..ed20ada7570 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -32,20 +32,20 @@ OR parser.add_argument("--compset", "-compset", required=True, help="(required) Specify a compset. " - "To see list of current compsets, use the utility manage_case in this directory") + "To see list of current compsets, use the utility query_case in this directory") parser.add_argument("--res", "-res", required=True, help="(required) Specify a model grid resolution. " - "To see list of current compsets, use the utility manage_case in this directory") + "To see list of current compsets, use the utility query_case in this directory") parser.add_argument("--machine", "-mach", help="Specify a machine. default: match NODENAME_REGEX in config_machines.xml " - "To see list of current machines, use the utility manage_case in this directory" + "To see list of current machines, use the utility query_case in this directory" ) parser.add_argument("--compiler", "-compiler", help="Specify a compiler. " - "To see list of supported compilers for each machine, use the utility manage_case in this directory") + "To see list of supported compilers for each machine, use the utility query_case in this directory") parser.add_argument("--ninst",default=1, help="Specify number of component instances" @@ -53,7 +53,7 @@ OR parser.add_argument("--mpilib", "-mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility manage_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") parser.add_argument("--project", "-project", diff --git a/scripts/create_test b/scripts/create_test index 35654578954..3fbe88c4ddd 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -124,7 +124,7 @@ OR parser.add_argument("--mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility manage_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") if model == "cesm": diff --git a/scripts/lib/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py index 351b40d254e..15643022c6d 100644 --- a/scripts/lib/CIME/XML/grids.py +++ b/scripts/lib/CIME/XML/grids.py @@ -113,7 +113,7 @@ def _read_config_grids_v1(self, name, compset): component_grids = self._get_component_grids(lname) return lname, component_grids expect (False, - "grid '%s' is not supported, use manage_case to determine supported grids " %name) + "grid '%s' is not supported, use query_case to determine supported grids " %name) def _read_config_grids_v2(self, name, compset): """ diff --git a/scripts/manage_case b/scripts/query_case similarity index 100% rename from scripts/manage_case rename to scripts/query_case diff --git a/tools/Readme.configure b/tools/Readme.configure index 36b310ee5f9..8bae506bedf 100644 --- a/tools/Readme.configure +++ b/tools/Readme.configure @@ -39,11 +39,11 @@ optional arguments: defaults to the current working directory. --compiler COMPILER, -compiler COMPILER Specify a compiler. To see list of supported compilers - for each machine, use the utility manage_case in this + for each machine, use the utility query_case in this directory --mpilib MPILIB, -mpilib MPILIB Specify the mpilib. To see list of supported mpilibs - for each machine, use the utility manage_case in this + for each machine, use the utility query_case in this directory. The default is the first listing in MPILIBS in config_machines.xml --clean Remove old Macros and env files before attempting to diff --git a/tools/configure b/tools/configure index 997b1d3e524..b6e4ed65424 100755 --- a/tools/configure +++ b/tools/configure @@ -57,14 +57,14 @@ def parse_command_line(args): parser.add_argument("--compiler", "-compiler", help="Specify a compiler. " - "To see list of supported compilers for each machine, use the utility manage_case in this directory") + "To see list of supported compilers for each machine, use the utility query_case in this directory") parser.add_argument("--mpilib", "-mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility manage_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") - parser.add_argument("--clean", action="store_true", + parser.add_argument("--clean", action="store_true", help="Remove old Macros and env files before attempting to create new ones") argcnt = len(args) @@ -135,7 +135,7 @@ def parse_command_line(args): if args.clean: - files = ["Macros.make", "Macros.cmake", "env_mach_specific.xml", ".env_mach_specific.sh", + files = ["Macros.make", "Macros.cmake", "env_mach_specific.xml", ".env_mach_specific.sh", ".env_mach_specific.csh", "Depends.%s"%compiler, "Depends.%s"%args.machine, "Depends.%s.%s"%(args.machine,compiler)] for file_ in files: From 7a85eaf456ce698524d286c25c9adf7d9dd35d72 Mon Sep 17 00:00:00 2001 From: Erich L Foster Date: Thu, 20 Apr 2017 09:41:35 -0600 Subject: [PATCH 2/5] Switched to query_config from query_case The last commit mistakenly used query_case instead of query_config. This commit fixes this issue. Fixes #1155 --- scripts/create_newcase | 10 +++++----- scripts/create_test | 2 +- scripts/lib/CIME/XML/grids.py | 2 +- scripts/{query_case => query_config} | 0 tools/Readme.configure | 4 ++-- tools/configure | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) rename scripts/{query_case => query_config} (100%) diff --git a/scripts/create_newcase b/scripts/create_newcase index ed20ada7570..995ae439bfa 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -32,20 +32,20 @@ OR parser.add_argument("--compset", "-compset", required=True, help="(required) Specify a compset. " - "To see list of current compsets, use the utility query_case in this directory") + "To see list of current compsets, use the utility query_config in this directory") parser.add_argument("--res", "-res", required=True, help="(required) Specify a model grid resolution. " - "To see list of current compsets, use the utility query_case in this directory") + "To see list of current compsets, use the utility query_config in this directory") parser.add_argument("--machine", "-mach", help="Specify a machine. default: match NODENAME_REGEX in config_machines.xml " - "To see list of current machines, use the utility query_case in this directory" + "To see list of current machines, use the utility query_config in this directory" ) parser.add_argument("--compiler", "-compiler", help="Specify a compiler. " - "To see list of supported compilers for each machine, use the utility query_case in this directory") + "To see list of supported compilers for each machine, use the utility query_config in this directory") parser.add_argument("--ninst",default=1, help="Specify number of component instances" @@ -53,7 +53,7 @@ OR parser.add_argument("--mpilib", "-mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_config in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") parser.add_argument("--project", "-project", diff --git a/scripts/create_test b/scripts/create_test index 3fbe88c4ddd..de6eebcd28e 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -124,7 +124,7 @@ OR parser.add_argument("--mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_config in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") if model == "cesm": diff --git a/scripts/lib/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py index 15643022c6d..a5ae9b397f9 100644 --- a/scripts/lib/CIME/XML/grids.py +++ b/scripts/lib/CIME/XML/grids.py @@ -113,7 +113,7 @@ def _read_config_grids_v1(self, name, compset): component_grids = self._get_component_grids(lname) return lname, component_grids expect (False, - "grid '%s' is not supported, use query_case to determine supported grids " %name) + "grid '%s' is not supported, use query_config to determine supported grids " %name) def _read_config_grids_v2(self, name, compset): """ diff --git a/scripts/query_case b/scripts/query_config similarity index 100% rename from scripts/query_case rename to scripts/query_config diff --git a/tools/Readme.configure b/tools/Readme.configure index 8bae506bedf..08084f8cf3f 100644 --- a/tools/Readme.configure +++ b/tools/Readme.configure @@ -39,11 +39,11 @@ optional arguments: defaults to the current working directory. --compiler COMPILER, -compiler COMPILER Specify a compiler. To see list of supported compilers - for each machine, use the utility query_case in this + for each machine, use the utility query_config in this directory --mpilib MPILIB, -mpilib MPILIB Specify the mpilib. To see list of supported mpilibs - for each machine, use the utility query_case in this + for each machine, use the utility query_config in this directory. The default is the first listing in MPILIBS in config_machines.xml --clean Remove old Macros and env files before attempting to diff --git a/tools/configure b/tools/configure index b6e4ed65424..81f5e81b881 100755 --- a/tools/configure +++ b/tools/configure @@ -57,11 +57,11 @@ def parse_command_line(args): parser.add_argument("--compiler", "-compiler", help="Specify a compiler. " - "To see list of supported compilers for each machine, use the utility query_case in this directory") + "To see list of supported compilers for each machine, use the utility query_config in this directory") parser.add_argument("--mpilib", "-mpilib", help="Specify the mpilib. " - "To see list of supported mpilibs for each machine, use the utility query_case in this directory. " + "To see list of supported mpilibs for each machine, use the utility query_config in this directory. " "The default is the first listing in MPILIBS in config_machines.xml") parser.add_argument("--clean", action="store_true", From 4f0a70ec2bacfba02a5a394055d9f86a466ed56a Mon Sep 17 00:00:00 2001 From: Erich L Foster Date: Tue, 25 Apr 2017 08:33:44 -0600 Subject: [PATCH 3/5] Update query_config as per #1190 Changed all arguments to short names. Updated printing for most requests, such as shorter compset output, ordered (alphabetical) compset printing, added 'all' option to compsets argument, added 'current' to machines argument but maintained previous behavior. Fixed issue where help didn't print when no arguments were provided. Also, fixed issue where a bogus value was giving to compsets and the error message printed wasn't very useful. No we print available components along with usage. Fixes #1190 --- scripts/lib/CIME/XML/component.py | 2 +- scripts/lib/CIME/XML/compsets.py | 5 +- scripts/lib/CIME/XML/machines.py | 2 +- scripts/query_config | 199 +++++++++++++++++++++++------- 4 files changed, 157 insertions(+), 51 deletions(-) diff --git a/scripts/lib/CIME/XML/component.py b/scripts/lib/CIME/XML/component.py index d4b8f8db8c9..57b79e145a1 100644 --- a/scripts/lib/CIME/XML/component.py +++ b/scripts/lib/CIME/XML/component.py @@ -108,6 +108,6 @@ def print_values(self): compsets[attrib] = text logger.info(" %s" %helptext) - for v in compsets.iteritems(): + for v in sorted(compsets.iteritems()): label, definition = v logger.info(" %20s : %s" %(label, definition)) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 2d96aaed3e6..70a0a6cf7f7 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -80,10 +80,11 @@ def get_value(self, name, attribute=None, resolved=False, subgroup=None): compsets[alias] = lname return compsets - def print_values(self): + def print_values(self, help=True): help_text = self.get_value(name="help") compsets_text = self.get_value("names") - logger.info(" %s " %help_text) + if help: + logger.info(" %s " %help_text) logger.info(" --------------------------------------") logger.info(" Compset Short Name: Compset Long Name ") diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index 67e37b6e973..d8f2655a03a 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -91,7 +91,7 @@ def list_available_machines(self): machines.append(mach) return machines - def probe_machine_name(self): + def probe_machine_name(self, warn=True): """ Find a matching regular expression for hostname in the NODENAME_REGEX field in the file. First match wins. diff --git a/scripts/query_config b/scripts/query_config index 40f6a8fc280..701ce22cbb4 100755 --- a/scripts/query_config +++ b/scripts/query_config @@ -2,14 +2,15 @@ from Tools.standard_script_setup import * -from CIME.utils import expect +from CIME.utils import expect, get_model from CIME.XML.files import Files from CIME.XML.component import Component from CIME.XML.compsets import Compsets from CIME.XML.grids import Grids -from CIME.XML.machines import Machines +#from CIME.XML.machines import Machines +import CIME.XML.machines -import argparse, doctest +import re, socket, argparse, doctest ############################################################################### def query_grids(long_output): @@ -18,7 +19,7 @@ def query_grids(long_output): files = Files() config_file = files.get_value("GRIDS_SPEC_FILE") expect(os.path.isfile(config_file), - "Cannot find config_file %s on disk" %config_file) + "Cannot find config_file {} on disk".format(config_file)) grids = Grids(config_file) if long_output: @@ -27,47 +28,71 @@ def query_grids(long_output): grids.print_values() ############################################################################### -def query_machines(): +def query_machines(current=False): ############################################################################### files = Files() config_file = files.get_value("MACHINES_SPEC_FILE") expect(os.path.isfile(config_file), - "Cannot find config_file %s on disk" %config_file) + "Cannot find config_file {} on disk".format(config_file)) # Provide a special machine name indicating no need for a machine name machines = Machines(config_file,machine="Query") - machines.print_values() + machines.print_values(current_only=current) ############################################################################### def query_compsets(name): ############################################################################### # Determine valid component values by checking the value attributes for COMPSETS_SPEC_FILE - files = Files() - components = files.get_components("COMPSETS_SPEC_FILE") + files, components = get_compsets() match_found = None - for component in components: - if component == name: - match_found = name - break + all = False + if re.search("^all$", name): # print all compsets + model = get_model() + match_found = name + all = True + else: + for component in components: + if component == name: + match_found = name + break # If name is not a valid argument - exit with error expect(match_found is not None, - "Invalid input argument %s, valid input arguments are %s" % (name, components)) + "Invalid input argument {}, valid input arguments are {}".format(name, components)) - # Determine the config_file for the target component - config_file = files.get_value("COMPSETS_SPEC_FILE", attribute={"component":name}) - expect((config_file), - "Cannot find any config_component.xml file for %s" %name) + if all: # print all compsets + for component in components: + # the all flag will only print available components + print_config(component, files, all=all) + else: + print_config(name, files) - # Check that file exists on disk - expect(os.path.isfile(config_file), - "Cannot find config_file %s on disk" %config_file) +def print_config(name, files, all=False): + ''' + print compsets associated with the component name, but if all is true only + print the details if the associated component is available + ''' + # Determine the config_file for the target component + config_file = files.get_value("COMPSETS_SPEC_FILE", attribute={"component":name}) + # only error out if we aren't printing all otherwise exit quitely + if not all: + expect((config_file), + "Cannot find any config_component.xml file for {}".format(name)) + + # Check that file exists on disk + expect(os.path.isfile(config_file), + "Cannot find config_file {} on disk".format(config_file)) + elif config_file is None or not os.path.isfile(config_file): + return + + print "\nActive component: {}".format(name) # Now parse the compsets file and write out the compset alias and longname as well as the help text # determine component xml content compsets = Compsets(config_file) - compsets.print_values() + # print compsets associated with component without help text + compsets.print_values(help=False) ############################################################################### def query_component(name): @@ -75,17 +100,14 @@ def query_component(name): # Determine the valid component classes (e.g. atm) for the driver/cpl # These are then stored in comps_array - files = Files() - infile = files.get_value("CONFIG_CPL_FILE") - config_drv = Component(infile) - components = config_drv.get_valid_model_components() + files, components = get_components() # Loop through the elements for each component class (in config_files.xml) # and see if there is a match for the the target component in the component attribute match_found = False valid_components = [] for comp in components: - string = "CONFIG_%s_FILE"%comp + string = "CONFIG_{}_FILE".format(comp) # determine all components in string components = files.get_components(string) @@ -97,16 +119,16 @@ def query_component(name): if config_file is not None: match_found = True expect(os.path.isfile(config_file), - "Cannot find config_file %s on disk" %config_file) + "Cannot find config_file {} on disk".format(config_file)) break # If name is not a valid argument - exit with error expect(match_found, - "Invalid input argument %s, valid input arguments are %s" % (name, valid_components)) + "Invalid input argument {}, valid input arguments are {}".format(name, valid_components)) # Check that file exists on disk, if not exit with error expect((config_file), - "Cannot find any config_component.xml file for %s" %name) + "Cannot find any config_component.xml file for {}".format(name)) # determine component xml content component = Component(config_file) @@ -118,49 +140,132 @@ def parse_command_line(args): cime_model = CIME.utils.get_model() - parser = argparse.ArgumentParser() + parser = ArgumentParser() CIME.utils.setup_standard_logging_options(parser) - parser.add_argument("--query-compsets-setby", - help="Query compsets that are set by the target component for %s model"%cime_model) + parser.add_argument("--compset", + help="Query compsets that are set by the target component for {} model".format(cime_model)) - parser.add_argument("--query-component-name", - help="Query component settings that are set by the target component for %s model"%cime_model) + parser.add_argument("--component", + help="Query component settings that are set by the target component for {} model".format(cime_model)) - parser.add_argument("--query-grids", action="store_true", - help="Query supported model grids for %s model" %cime_model) + parser.add_argument("--grids", action="store_true", + help="Query supported model grids for {} model".format(cime_model)) - parser.add_argument("--grids-alias", - help="Query model grids for input grid alias for %s model - not implemented yet" %cime_model) +# what was the intent of this argument? +# parser.add_argument("--grids-alias", +# help="Query model grids for input grid alias for {} model - not implemented yet".format(cime_model)) - parser.add_argument("--query-machines", action="store_true", - help="Query supported machines for %s model" %cime_model) + parser.add_argument("--machines", nargs='?', action='store', const=True, choices=['current', 'empty'], + help="Query supported machines for {} model".format(cime_model)) parser.add_argument("--long", action="store_true", help="Provide long output for queries") args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) + # make sure at least one argument has been passed + if not (args.grids or args.compset or args.component or args.machines): + parser.print_help(sys.stderr) + + CIME.utils.handle_standard_logging_options(args) + return args +def get_compsets(): + ''' + Determine valid component values by checking the value attributes for COMPSETS_SPEC_FILE + ''' + files = Files() + return files, files.get_components("COMPSETS_SPEC_FILE") + +def get_components(): + ''' + Determine the valid component classes (e.g. atm) for the driver/cpl + These are then stored in comps_array + ''' + files = Files() + infile = files.get_value("CONFIG_CPL_FILE") + config_drv = Component(infile) + return files, config_drv.get_valid_model_components() + +# we override the error message from ArgumentParser to have a more helpful +# message in the case of missing arguments +class ArgumentParser(argparse.ArgumentParser): + + def error(self, message): + self.print_usage(sys.stderr) + # missing argument + if "expected one argument" in message: + if "compset" in message: + files, components = get_compsets() + self.exit(2, '{}: error: {}\nValid input arguments are {}\n' + .format(self.prog, message, components)) + elif "component" in message: + files, components = get_components() + self.exit(2, '{}: error: {}\nValid input arguments are {}\n' + .format(self.prog, message, components)) + # for all other errors + self.exit(2, '{}: error: {}\n'.format(self.prog, message)) + +# we overide print_values from Machines to add current in machine description +class Machines(CIME.XML.machines.Machines): + + def print_values(self, current_only=False): + # write out machines + if current_only and not self.probe_machine_name(warn=False): + print "Current machine is not listed in config file: {}".format(config_file) + else: + machines = self.get_nodes(nodename="machine") + print "Machines" + for machine in machines: + name = machine.get("MACH") + desc = machine.find("DESC") + os_ = machine.find("OS") + compilers = machine.find("COMPILERS") + max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") + pes_per_node = machine.find("PES_PER_NODE") + + current_machine = self.probe_machine_name(warn=False) + if current_machine and current_machine in name: + print " {} : {} ".format(name + " (current)", desc.text) + print " os ", os_.text + print " compilers ",compilers.text + if pes_per_node is not None: + print " pes/node ",pes_per_node.text + if max_tasks_per_node is not None: + print " max_tasks/node ",max_tasks_per_node.text + elif not current_only: + print " {} : {} ".format(name, desc.text) + print " os ", os_.text + print " compilers ",compilers.text + if pes_per_node is not None: + print " pes/node ",pes_per_node.text + if max_tasks_per_node is not None: + print " max_tasks/node ",max_tasks_per_node.text + ############################################################################### def _main_func(): ############################################################################### args = parse_command_line(sys.argv) - if args.query_grids: + if args.grids: query_grids(long_output=args.long) - if args.query_compsets_setby: - query_compsets(name=args.query_compsets_setby) + if args.compset: + query_compsets(name=args.compset) - if args.query_component_name: - query_component(args.query_component_name) + if args.component: + query_component(args.component) - if args.query_machines: + # this order is required since booleans are not iterable + if args.machines is not None and args.machines is True: query_machines() + elif args.machines is not None and 'current' in args.machines: + query_machines(current=True) + ############################################################################### From 20eee71358ba3b3b7964c5871800691d7a439643 Mon Sep 17 00:00:00 2001 From: Erich L Foster Date: Tue, 2 May 2017 13:59:57 -0600 Subject: [PATCH 4/5] Added Ability to Query Single Machine In this commit-- added the ability to give machine name as a query. Additionally, fixed some the argument descriptions. Also, fixed argument descriptions to be more descriptive on choices. Test suite: `scripts_regression_tests.py` and ``` ./query_config ./query_config --grids ./query_config --machines ./query_config --machines current ./query_config --machines redsky ./query_config --machines edison ./query_config --machines skybridge ./query_config --compsets ./query_config --compsets all ./query_config --compsets drv ./query_config --component ``` Test baseline: Test namelist changes: Test status: bit for bit Fixes #1155, #1190 User interface changes?: manage_case -> query_config --compset -> --compsets --machine -> --machines added 'all' option for --compsets added 'current' and to --machines Code review: @billsacks, @jgfouca --- scripts/lib/CIME/XML/compsets.py | 10 ++-- scripts/lib/CIME/XML/machines.py | 17 +++--- scripts/query_config | 94 +++++++++++++++++++++++--------- 3 files changed, 83 insertions(+), 38 deletions(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 70a0a6cf7f7..fe747ed19b7 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -42,7 +42,7 @@ def get_compset_match(self, name): user_mods = user_mods_node.text else: user_mods = None - logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) + logger.debug("Found node match with alias: {} and lname: {}".format(alias, lname)) return (lname, alias, science_support, user_mods) return (None, None, [False], None) @@ -72,7 +72,7 @@ def get_value(self, name, attribute=None, resolved=False, subgroup=None): nodes = self.get_nodes(nodename="compset") for node in nodes: for child in node: - logger.debug ("Here child is %s with value %s"%(child.tag,child.text)) + logger.debug ("Here child is {} with value {}".format(child.tag,child.text)) if child.tag == "alias": alias = child.text if child.tag == "lname": @@ -84,11 +84,11 @@ def print_values(self, help=True): help_text = self.get_value(name="help") compsets_text = self.get_value("names") if help: - logger.info(" %s " %help_text) + logger.info(" {} ".format(help_text)) logger.info(" --------------------------------------") - logger.info(" Compset Short Name: Compset Long Name ") + logger.info(" Compset Alias: Compset Long Name ") logger.info(" --------------------------------------") for v in compsets_text.iteritems(): label, definition = v - logger.info(" %20s : %s" %(label, definition)) + logger.info(" {:20} : {}".format(label, definition)) diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index d8f2655a03a..41eefae1760 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -29,7 +29,7 @@ def __init__(self, infile=None, files=None, machine=None): if infile is None: infile = files.get_value("MACHINES_SPEC_FILE") schema = files.get_schema("MACHINES_SPEC_FILE") - logger.debug("Verifying using schema %s"%schema) + logger.debug("Verifying using schema {}".format(schema)) self.machines_dir = os.path.dirname(infile) @@ -38,7 +38,7 @@ def __init__(self, infile=None, files=None, machine=None): # Append the contents of $HOME/.cime/config_machines.xml if it exists # This could cause problems if node matchs are repeated when only one is expected local_infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml") - logger.debug("Infile: %s" , local_infile) + logger.debug("Infile: {}" , local_infile) if os.path.exists(local_infile): GenericXML.read(self, local_infile, schema) @@ -48,7 +48,7 @@ def __init__(self, infile=None, files=None, machine=None): else: machine = self.probe_machine_name() - expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile)) + expect(machine is not None, "Could not initialize machine object from {} or {}".format(infile, local_infile)) self.set_machine(machine) def get_machines_dir(self): @@ -113,7 +113,8 @@ def probe_machine_name(self, warn=True): names_not_found_quoted = ["'" + name + "'" for name in names_not_found] names_not_found_str = ' or '.join(names_not_found_quoted) - logger.warning("Could not find machine match for %s" % names_not_found_str) + if warn: + logger.warning("Could not find machine match for {}".format(names_not_found_str)) return machine @@ -136,7 +137,7 @@ def _probe_machine_name_one_guess(self, nametomatch): logger.debug("machine regex string is " + regex_str) regex = re.compile(regex_str) if regex.match(nametomatch): - logger.debug("Found machine: %s matches %s" % (machtocheck, nametomatch)) + logger.debug("Found machine: {} matches {}".format(machtocheck, nametomatch)) machine = machtocheck break @@ -158,7 +159,7 @@ def set_machine(self, machine): self.machine = machine elif self.machine != machine or self.machine_node is None: self.machine_node = self.get_optional_node("machine", {"MACH" : machine}) - expect(self.machine_node is not None, "No machine %s found" % machine) + expect(self.machine_node is not None, "No machine {} found".format(machine)) self.machine = machine return machine @@ -275,7 +276,7 @@ def has_batch_system(self): batch_system = self.get_optional_node("BATCH_SYSTEM", root=self.machine_node) if batch_system is not None: result = (batch_system.text is not None and batch_system.text != "none") - logger.debug("Machine %s has batch: %s" % (self.machine, result)) + logger.debug("Machine {} has batch: {}".format(self.machine, result)) return result def get_suffix(self, suffix_type): @@ -299,7 +300,7 @@ def print_values(self): max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") pes_per_node = machine.find("PES_PER_NODE") - print " %s : %s "% (name , desc.text) + print " {} : {} ".format(name , desc.text) print " os ", os_.text print " compilers ",compilers.text if pes_per_node is not None: diff --git a/scripts/query_config b/scripts/query_config index 701ce22cbb4..28c58fa118f 100755 --- a/scripts/query_config +++ b/scripts/query_config @@ -28,7 +28,7 @@ def query_grids(long_output): grids.print_values() ############################################################################### -def query_machines(current=False): +def query_machines(machine_name=None): ############################################################################### files = Files() @@ -36,8 +36,8 @@ def query_machines(current=False): expect(os.path.isfile(config_file), "Cannot find config_file {} on disk".format(config_file)) # Provide a special machine name indicating no need for a machine name - machines = Machines(config_file,machine="Query") - machines.print_values(current_only=current) + machines = Machines(config_file, machine="Query") + machines.print_values(machine_name=machine_name) ############################################################################### def query_compsets(name): @@ -144,11 +144,26 @@ def parse_command_line(args): CIME.utils.setup_standard_logging_options(parser) - parser.add_argument("--compset", - help="Query compsets that are set by the target component for {} model".format(cime_model)) + files, components = get_compsets() + valid_components = ['all'] + for item in components: + valid_components.append(item) - parser.add_argument("--component", - help="Query component settings that are set by the target component for {} model".format(cime_model)) + parser.add_argument("--compsets", nargs='?', const='all', choices=valid_components, + help="Query compsets corresponding to the target component for the {} model".format(cime_model)) + + files, components = get_components() + # Loop through the elements for each component class (in config_files.xml) + valid_components = [] + for comp in components: + string = "CONFIG_{}_FILE".format(comp) + + # determine all components in string + components = files.get_components(string) + for item in components: + valid_components.append(item) + parser.add_argument("--component", choices=valid_components, + help="Query component settings corresponding to the target component for {} model".format(cime_model)) parser.add_argument("--grids", action="store_true", help="Query supported model grids for {} model".format(cime_model)) @@ -157,8 +172,18 @@ def parse_command_line(args): # parser.add_argument("--grids-alias", # help="Query model grids for input grid alias for {} model - not implemented yet".format(cime_model)) - parser.add_argument("--machines", nargs='?', action='store', const=True, choices=['current', 'empty'], - help="Query supported machines for {} model".format(cime_model)) + config_file = files.get_value("MACHINES_SPEC_FILE") + expect(os.path.isfile(config_file), + "Cannot find config_file {} on disk".format(config_file)) + machines = Machines(config_file, machine="Query") + machines = machines.get_nodes(nodename="machine") + machine_names = ['current'] + for machine in machines: + machine_names.append(machine.get("MACH")) + parser.add_argument("--machines", nargs='?', action='store', const=True, choices=machine_names, + help="Query supported machines for {} model." + " If option is left empty then all machines are listed," + " if current is giving the current machine details are listed.".format(cime_model)) parser.add_argument("--long", action="store_true", help="Provide long output for queries") @@ -166,7 +191,7 @@ def parse_command_line(args): args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) # make sure at least one argument has been passed - if not (args.grids or args.compset or args.component or args.machines): + if not (args.grids or args.compsets or args.component or args.machines): parser.print_help(sys.stderr) CIME.utils.handle_standard_logging_options(args) @@ -204,21 +229,40 @@ class ArgumentParser(argparse.ArgumentParser): .format(self.prog, message, components)) elif "component" in message: files, components = get_components() + # Loop through the elements for each component class (in config_files.xml) + valid_components = [] + for comp in components: + string = "CONFIG_{}_FILE".format(comp) + + # determine all components in string + components = files.get_components(string) + for item in components: + valid_components.append(item) self.exit(2, '{}: error: {}\nValid input arguments are {}\n' - .format(self.prog, message, components)) + .format(self.prog, message, valid_components)) # for all other errors self.exit(2, '{}: error: {}\n'.format(self.prog, message)) # we overide print_values from Machines to add current in machine description class Machines(CIME.XML.machines.Machines): - def print_values(self, current_only=False): - # write out machines - if current_only and not self.probe_machine_name(warn=False): - print "Current machine is not listed in config file: {}".format(config_file) + def print_values(self, machine_name=None): + # set flag to look for single machine + if machine_name is not None: + single_machine = True + if machine_name == 'current': + machine_name = self.probe_machine_name(warn=False) else: + single_machine = False + + # if we can't find the specified machine + if single_machine and machine_name is None: + files = Files() + config_file = files.get_value("MACHINES_SPEC_FILE") + print "Machine is not listed in config file: {}".format(config_file) + else: # write out machines machines = self.get_nodes(nodename="machine") - print "Machines" + print "Machine(s)" for machine in machines: name = machine.get("MACH") desc = machine.find("DESC") @@ -228,15 +272,16 @@ class Machines(CIME.XML.machines.Machines): pes_per_node = machine.find("PES_PER_NODE") current_machine = self.probe_machine_name(warn=False) - if current_machine and current_machine in name: - print " {} : {} ".format(name + " (current)", desc.text) + if not single_machine: + name += " (current)" if current_machine and current_machine in name else "" + print " {} : {} ".format(name, desc.text) print " os ", os_.text print " compilers ",compilers.text if pes_per_node is not None: print " pes/node ",pes_per_node.text if max_tasks_per_node is not None: print " max_tasks/node ",max_tasks_per_node.text - elif not current_only: + elif single_machine and machine_name in name: print " {} : {} ".format(name, desc.text) print " os ", os_.text print " compilers ",compilers.text @@ -254,17 +299,16 @@ def _main_func(): if args.grids: query_grids(long_output=args.long) - if args.compset: - query_compsets(name=args.compset) + if args.compsets: + query_compsets(name=args.compsets) if args.component: query_component(args.component) - # this order is required since booleans are not iterable - if args.machines is not None and args.machines is True: + if args.machines is not None and args.machines is True: # print all machines query_machines() - elif args.machines is not None and 'current' in args.machines: - query_machines(current=True) + elif args.machines is not None: # print the selected machine + query_machines(machine_name=args.machines) ############################################################################### From 1bd5f7b7186e9877bd527971cca1c1f0684b2282 Mon Sep 17 00:00:00 2001 From: Erich L Foster Date: Fri, 5 May 2017 12:26:04 -0600 Subject: [PATCH 5/5] Added Changes as per @billsacks * added the ability to print all compsets with argument * added the ability to print all components with and without argument * added the ability to print a specified single machine * many config_component.xml files didn't have a help string, which caused --components all not to work. * fixed some typos and gramatical errors Test suite:`scripts_regression_tests.py` and ``` ./query_config ./query_config --grids ./query_config --machines ./query_config --machines current ./query_config --machines redsky ./query_config --machines edison ./query_config --machines skybridge ./query_config --compsets ./query_config --compsets all ./query_config --compsets drv ./query_config --component ``` Test baseline: Test namelist changes: Test status: bit for bit Fixes #1155, #1190 User interface changes?: * manage_case -> query_config * --compset -> --compsets * --machine -> --machines * added 'all' option for --compsets * added 'current' to --machines * added to --machines * added 'all' to --component * default for --component and --compsets is 'all' Code review: @billsacks, @jgfouca, @jedwards4b, @rljacob --- scripts/query_config | 85 ++++++++++++------- .../satm/cime_config/config_component.xml | 6 ++ .../sesp/cime_config/config_component.xml | 6 ++ .../sglc/cime_config/config_component.xml | 6 ++ .../sice/cime_config/config_component.xml | 6 ++ .../slnd/cime_config/config_component.xml | 6 ++ .../socn/cime_config/config_component.xml | 6 ++ .../srof/cime_config/config_component.xml | 6 ++ .../swav/cime_config/config_component.xml | 6 ++ .../xatm/cime_config/config_component.xml | 6 ++ .../xglc/cime_config/config_component.xml | 6 ++ .../xice/cime_config/config_component.xml | 6 ++ .../xlnd/cime_config/config_component.xml | 6 ++ .../xocn/cime_config/config_component.xml | 6 ++ .../xrof/cime_config/config_component.xml | 6 ++ .../xwav/cime_config/config_component.xml | 6 ++ 16 files changed, 144 insertions(+), 31 deletions(-) diff --git a/scripts/query_config b/scripts/query_config index 28c58fa118f..994fa9d0975 100755 --- a/scripts/query_config +++ b/scripts/query_config @@ -1,4 +1,10 @@ #!/usr/bin/env python +""" +Displays information about available compsets, component settings, grids and/or +machines. Typically run with one of the arguments --compsets, --settings, +--grids or --machines; if you specify more than one of these arguments, +information will be listed for each. +""" from Tools.standard_script_setup import * @@ -28,7 +34,7 @@ def query_grids(long_output): grids.print_values() ############################################################################### -def query_machines(machine_name=None): +def query_machines(machine_name='all'): ############################################################################### files = Files() @@ -64,11 +70,11 @@ def query_compsets(name): if all: # print all compsets for component in components: # the all flag will only print available components - print_config(component, files, all=all) + print_compset(component, files, all=all) else: - print_config(name, files) + print_compset(name, files) -def print_config(name, files, all=False): +def print_compset(name, files, all=False): ''' print compsets associated with the component name, but if all is true only print the details if the associated component is available @@ -94,8 +100,19 @@ def print_config(name, files, all=False): # print compsets associated with component without help text compsets.print_values(help=False) +def query_all_components(): + files, components = get_components() + # Loop through the elements for each component class (in config_files.xml) + for comp in components: + string = "CONFIG_{}_FILE".format(comp) + + # determine all components in string + components = files.get_components(string) + for item in components: + query_component(item, all=True) + ############################################################################### -def query_component(name): +def query_component(name, all=False): ############################################################################### # Determine the valid component classes (e.g. atm) for the driver/cpl @@ -118,10 +135,15 @@ def query_component(name): config_file = files.get_value(string, attribute={"component":name}) if config_file is not None: match_found = True - expect(os.path.isfile(config_file), - "Cannot find config_file {} on disk".format(config_file)) + config_exists = os.path.isfile(config_file) break + if not all and not config_exists: + expect(config_exists, + "Cannot find config_file {} on disk".format(config_file)) + elif all and not config_exists: + print "WARNING: Couldn't find config_file {} on disk".format(config_file) + return # If name is not a valid argument - exit with error expect(match_found, "Invalid input argument {}, valid input arguments are {}".format(name, valid_components)) @@ -135,12 +157,12 @@ def query_component(name): component.print_values() ############################################################################### -def parse_command_line(args): +def parse_command_line(args, description): ############################################################################### cime_model = CIME.utils.get_model() - parser = ArgumentParser() + parser = ArgumentParser(description=description) CIME.utils.setup_standard_logging_options(parser) @@ -150,11 +172,12 @@ def parse_command_line(args): valid_components.append(item) parser.add_argument("--compsets", nargs='?', const='all', choices=valid_components, - help="Query compsets corresponding to the target component for the {} model".format(cime_model)) + help="Query compsets corresponding to the target component for the {} model." + " If no component is given, lists compsets defined by all components".format(cime_model)) files, components = get_components() # Loop through the elements for each component class (in config_files.xml) - valid_components = [] + valid_components = ['all'] for comp in components: string = "CONFIG_{}_FILE".format(comp) @@ -162,11 +185,12 @@ def parse_command_line(args): components = files.get_components(string) for item in components: valid_components.append(item) - parser.add_argument("--component", choices=valid_components, - help="Query component settings corresponding to the target component for {} model".format(cime_model)) + parser.add_argument("--components", nargs='?', const='all', choices=valid_components, + help="Query component settings corresponding to the target component for {} model." + " If no component is given, lists settings defined by all components".format(cime_model)) parser.add_argument("--grids", action="store_true", - help="Query supported model grids for {} model".format(cime_model)) + help="Query supported model grids for {} model.".format(cime_model)) # what was the intent of this argument? # parser.add_argument("--grids-alias", @@ -177,13 +201,13 @@ def parse_command_line(args): "Cannot find config_file {} on disk".format(config_file)) machines = Machines(config_file, machine="Query") machines = machines.get_nodes(nodename="machine") - machine_names = ['current'] + machine_names = ['all', 'current'] for machine in machines: machine_names.append(machine.get("MACH")) - parser.add_argument("--machines", nargs='?', action='store', const=True, choices=machine_names, + parser.add_argument("--machines", nargs='?', const='all', choices=machine_names, help="Query supported machines for {} model." " If option is left empty then all machines are listed," - " if current is giving the current machine details are listed.".format(cime_model)) + " if current is given the current machine details are listed.".format(cime_model)) parser.add_argument("--long", action="store_true", help="Provide long output for queries") @@ -191,11 +215,9 @@ def parse_command_line(args): args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) # make sure at least one argument has been passed - if not (args.grids or args.compsets or args.component or args.machines): + if not (args.grids or args.compsets or args.components or args.machines): parser.print_help(sys.stderr) - CIME.utils.handle_standard_logging_options(args) - return args def get_compsets(): @@ -246,9 +268,9 @@ class ArgumentParser(argparse.ArgumentParser): # we overide print_values from Machines to add current in machine description class Machines(CIME.XML.machines.Machines): - def print_values(self, machine_name=None): + def print_values(self, machine_name='all'): # set flag to look for single machine - if machine_name is not None: + if 'all' not in machine_name: single_machine = True if machine_name == 'current': machine_name = self.probe_machine_name(warn=False) @@ -291,27 +313,28 @@ class Machines(CIME.XML.machines.Machines): print " max_tasks/node ",max_tasks_per_node.text ############################################################################### -def _main_func(): +def _main_func(description): ############################################################################### - args = parse_command_line(sys.argv) + args = parse_command_line(sys.argv, description) if args.grids: query_grids(long_output=args.long) - if args.compsets: + if args.compsets is not None: query_compsets(name=args.compsets) - if args.component: - query_component(args.component) + if args.components is not None: + if re.search("^all$", args.components): # print all compsets + query_all_components() + else: + query_component(args.components) - if args.machines is not None and args.machines is True: # print all machines - query_machines() - elif args.machines is not None: # print the selected machine + if args.machines is not None: query_machines(machine_name=args.machines) ############################################################################### if __name__ == "__main__": - _main_func() + _main_func(__doc__) diff --git a/src/components/stub_comps/satm/cime_config/config_component.xml b/src/components/stub_comps/satm/cime_config/config_component.xml index eabe56dbc1c..64e9c6476b5 100644 --- a/src/components/stub_comps/satm/cime_config/config_component.xml +++ b/src/components/stub_comps/satm/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub atm component + + ========================================= + SATM naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/sesp/cime_config/config_component.xml b/src/components/stub_comps/sesp/cime_config/config_component.xml index 0290f9100de..0edf9c2c38e 100644 --- a/src/components/stub_comps/sesp/cime_config/config_component.xml +++ b/src/components/stub_comps/sesp/cime_config/config_component.xml @@ -18,4 +18,10 @@ Stub esp component + + ========================================= + SESP naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/sglc/cime_config/config_component.xml b/src/components/stub_comps/sglc/cime_config/config_component.xml index 85e78d97b54..ce3e704527e 100644 --- a/src/components/stub_comps/sglc/cime_config/config_component.xml +++ b/src/components/stub_comps/sglc/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub glc component + + ========================================= + SGLC naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/sice/cime_config/config_component.xml b/src/components/stub_comps/sice/cime_config/config_component.xml index 987b15fdf2a..5e5ebc08e89 100644 --- a/src/components/stub_comps/sice/cime_config/config_component.xml +++ b/src/components/stub_comps/sice/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub ice component + + ========================================= + SICE naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/slnd/cime_config/config_component.xml b/src/components/stub_comps/slnd/cime_config/config_component.xml index d0b4639e0ce..ece4b11ee76 100644 --- a/src/components/stub_comps/slnd/cime_config/config_component.xml +++ b/src/components/stub_comps/slnd/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub lnd component + + ========================================= + SLND naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/socn/cime_config/config_component.xml b/src/components/stub_comps/socn/cime_config/config_component.xml index daf1d2c9ffb..be243eedc0e 100644 --- a/src/components/stub_comps/socn/cime_config/config_component.xml +++ b/src/components/stub_comps/socn/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub ocn component + + ========================================= + SOCN naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/srof/cime_config/config_component.xml b/src/components/stub_comps/srof/cime_config/config_component.xml index 92bdcca611b..10b13898d05 100644 --- a/src/components/stub_comps/srof/cime_config/config_component.xml +++ b/src/components/stub_comps/srof/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub river component + + ========================================= + SROF naming conventions in compset name + ========================================= + + diff --git a/src/components/stub_comps/swav/cime_config/config_component.xml b/src/components/stub_comps/swav/cime_config/config_component.xml index 8b5be883ed9..eb03a072775 100644 --- a/src/components/stub_comps/swav/cime_config/config_component.xml +++ b/src/components/stub_comps/swav/cime_config/config_component.xml @@ -17,4 +17,10 @@ Stub wave component + + ========================================= + SWAV naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xatm/cime_config/config_component.xml b/src/components/xcpl_comps/xatm/cime_config/config_component.xml index a1ad59fe62d..6635e3b53b2 100644 --- a/src/components/xcpl_comps/xatm/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xatm/cime_config/config_component.xml @@ -17,5 +17,11 @@ Dead atm component + + ========================================= + XATM naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xglc/cime_config/config_component.xml b/src/components/xcpl_comps/xglc/cime_config/config_component.xml index 9535d9a58f2..c4c42df5171 100644 --- a/src/components/xcpl_comps/xglc/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xglc/cime_config/config_component.xml @@ -17,4 +17,10 @@ Dead land-ice component + + ========================================= + XGLC naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xice/cime_config/config_component.xml b/src/components/xcpl_comps/xice/cime_config/config_component.xml index 34062742afd..aa85cf23dc4 100644 --- a/src/components/xcpl_comps/xice/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xice/cime_config/config_component.xml @@ -17,4 +17,10 @@ Dead ice component + + ========================================= + XICE naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xlnd/cime_config/config_component.xml b/src/components/xcpl_comps/xlnd/cime_config/config_component.xml index 792c3b28bcf..c9d73c4f143 100644 --- a/src/components/xcpl_comps/xlnd/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xlnd/cime_config/config_component.xml @@ -17,4 +17,10 @@ Dead land component + + ========================================= + XLND naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xocn/cime_config/config_component.xml b/src/components/xcpl_comps/xocn/cime_config/config_component.xml index 755f010d600..7fe9717a099 100644 --- a/src/components/xcpl_comps/xocn/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xocn/cime_config/config_component.xml @@ -17,4 +17,10 @@ Dead ocean component + + ========================================= + XOCN naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xrof/cime_config/config_component.xml b/src/components/xcpl_comps/xrof/cime_config/config_component.xml index 0daa65009f9..06f61ecdcd4 100644 --- a/src/components/xcpl_comps/xrof/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xrof/cime_config/config_component.xml @@ -30,4 +30,10 @@ Dead river component + + ========================================= + XROF naming conventions in compset name + ========================================= + + diff --git a/src/components/xcpl_comps/xwav/cime_config/config_component.xml b/src/components/xcpl_comps/xwav/cime_config/config_component.xml index 02c6b7eae5b..62bfbfa7255 100644 --- a/src/components/xcpl_comps/xwav/cime_config/config_component.xml +++ b/src/components/xcpl_comps/xwav/cime_config/config_component.xml @@ -17,4 +17,10 @@ Dead wave component + + ========================================= + XWAV naming conventions in compset name + ========================================= + +