Introduce multiple default settings files

The CESM 2.1 release will use CMIP6 tunings by default, but needs to be able to
reproduce the CESM 2.0 results as well. To do that, we introduced multiple
default settings files and a mechanism to select which is used. The default is
to use settings_latest.json, but there is also a settings_cesm2.0.json file.

default_diagnostics has also been renamed to diagnostics_latest, but at this
time MARBL does not support multiple possible default diagnostic files.

MARBL_tools/MARBL_generate_diagnostics_file.py

@@ -25,11 +25,11 @@
   -f DEFAULT_SETTINGS_FILE, --default_settings_file DEFAULT_SETTINGS_FILE
                         Location of JSON-formatted MARBL settings
                         configuration file (default: $MARBLROOT/
-                        autogenerated_src/default_settings.json)
+                        defaults/json/settings_latest.json)
   -j DEFAULT_DIAGNOSTICS_FILE, --default_diagnostics_file DEFAULT_DIAGNOSTICS_FILE
                         Location of JSON-formatted MARBL diagnostics
                         configuration file (default: $MARBLROOT/
-                        autogenerated_src/default_diagnostics.json)
+                        defaults/json/diagnostics_latest.json)
   -s {GCM,settings_file}, --saved_state_vars_source {GCM,settings_file}
                         Source of initial value for saved state vars that can
                         come from GCM or settings file (default:
@@ -102,14 +102,14 @@ def _parse_args(marbl_root):
     parser = argparse.ArgumentParser(description="Generate a MARBL settings file from a JSON file",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    # Command line argument to point to JSON settings file (default is $MARBLROOT/src/default_settings.json)
+    # Command line argument to point to JSON settings file (default is $MARBLROOT/defaults/json/settings_latest.json)
     parser.add_argument('-f', '--default_settings_file', action='store', dest='default_settings_file',
-                        default=os.path.join(marbl_root, 'autogenerated_src', 'default_settings.json'),
+                        default=os.path.join(marbl_root, 'defaults', 'json', 'settings_latest.json'),
                         help='Location of JSON-formatted MARBL settings configuration file')
 
-    # Command line argument to point to JSON diagnostics file (default is $MARBLROOT/src/default_diagnostics.json)
+    # Command line argument to point to JSON diagnostics file (default is $MARBLROOT/defaults/json/diagnostics_latest.json)
     parser.add_argument('-j', '--default_diagnostics_file', action='store', dest='default_diagnostics_file',
-                        default=os.path.join(marbl_root, 'autogenerated_src', 'default_diagnostics.json'),
+                        default=os.path.join(marbl_root, 'defaults', 'json', 'diagnostics_latest.json'),
                         help='Location of JSON-formatted MARBL diagnostics configuration file')
 
     # Is the GCM providing initial bury coefficients via saved state?

MARBL_tools/MARBL_generate_settings_file.py

@@ -23,7 +23,7 @@
   -h, --help            show this help message and exit
   -f DEFAULT_SETTINGS_FILE, --default_settings_file DEFAULT_SETTINGS_FILE
                         Location of JSON-formatted MARBL configuration file
-                        (default: $MARBLROOT/autogenerated_src/default_settings.json)
+                        (default: $MARBLROOT/defaults/json/settings_latest.json)
   -s {GCM,settings_file}, --saved_state_vars_source {GCM,settings_file}
                         Source of initial value for saved state vars that can
                         come from GCM or settings file (default:
@@ -63,9 +63,9 @@ def _parse_args(marbl_root):
     parser = argparse.ArgumentParser(description="Generate a MARBL settings file from a JSON file",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    # Command line argument to point to JSON file (default is $MARBLROOT/src/default_settings.json)
+    # Command line argument to point to JSON file (default is $MARBLROOT/defaults/json/settings_latest.json)
     parser.add_argument('-f', '--default_settings_file', action='store', dest='default_settings_file',
-                        default=os.path.join(marbl_root, 'autogenerated_src', 'default_settings.json'),
+                        default=os.path.join(marbl_root, 'defaults', 'json', 'settings_latest.json'),
                         help='Location of JSON-formatted MARBL configuration file')
 
     # Is the GCM providing initial bury coefficients via saved state?

MARBL_tools/yaml_to_json.py

@@ -4,114 +4,126 @@
 YAML support is not part of the Python standard library, so it is possible that
 users may find themselves running MARBL scripts in an environment that does not
 have PyYAML. To support those users, MARBL relies soly on JSON versions of
-default_settings.yaml and default_diagnostics.yaml which should be generated by
-this script if either YAML file is changed.
+settings_*.yaml and diagnostics_*.yaml which should be generated by this script
+if either YAML file is changed.
 
 Developers MUST update the YAML and then regenerate the JSON file(s); commits that
 change the JSON file(s) alone will not be accepted.
 
-usage: yaml_to_json.py [-h] [-s SETTINGS_FILE] [-d DIAGS_FILE] [-o OUTPUT_DIR]
+usage: yaml_to_json.py [-h] [-y YAML_FILES [YAML_FILES ...]] [-o OUTPUT_DIR]
 
-Convert MARBL's YAML files to JSON
+Convert all of MARBL's YAML files to JSON
 
 optional arguments:
   -h, --help            show this help message and exit
-  -s SETTINGS_FILE, --settings_file SETTINGS_FILE
-                        Location of YAML-formatted MARBL settings file to
-                        convert to JSON (default:
-                        $MARBLROOT/src/default_settings.yaml)
-  -d DIAGS_FILE, --diags_file DIAGS_FILE
-                        Location of YAML-formatted MARBL diagnostics file to
-                        convert to JSON (default:
-                        $MARBLROOT/src/default_diagnostics.yaml)
+  -y YAML_FILES [YAML_FILES ...], --yaml_files YAML_FILES [YAML_FILES ...]
+                        List of files to convert (default:
+                        ['$MARBLROOT/defaults/settings_cesm2.0.yaml',
+                        '$MARBLROOT/defaults/diagnostics_latest.yaml',
+                        '$MARBLROOT/defaults/settings_latest.yaml'])
   -o OUTPUT_DIR, --output_dir OUTPUT_DIR
                         Directory where JSON file(s) will be created (default:
-                        $MARBLROOT/autogenerated_src)
-
+                        $MARBLROOT/defaults/json)
 
 """
 
+import sys, os, logging
+
 #######################################
 
 def _parse_args(marbl_root):
     """ Parse command line arguments
     """
 
     import argparse
-    import os
 
-    parser = argparse.ArgumentParser(description="Convert MARBL's YAML files to JSON",
+    parser = argparse.ArgumentParser(description="Convert all of MARBL's YAML files to JSON",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('-s', '--settings_file', action='store', dest='settings_file',
-                        default=os.path.join(marbl_root, 'src', 'default_settings.yaml'),
-                        help='Location of YAML-formatted MARBL settings file to convert to JSON')
-
-    parser.add_argument('-d', '--diags_file', action='store', dest='diags_file',
-                        default=os.path.join(marbl_root, 'src', 'default_diagnostics.yaml'),
-                        help='Location of YAML-formatted MARBL diagnostics file to convert to JSON')
+    parser.add_argument('-y', '--yaml_files', nargs='+', action='store', dest='yaml_files',
+                        default=[os.path.join(yaml_dir,file) for file in os.listdir(yaml_dir) if file.endswith("yaml")],
+                        help="List of files to convert")
 
     parser.add_argument('-o', '--output_dir', action='store', dest='output_dir',
-                        default=os.path.join(marbl_root, 'autogenerated_src'),
+                        default=os.path.join(marbl_root, 'defaults', 'json'),
                         help="Directory where JSON file(s) will be created")
 
     return parser.parse_args()
 
 #######################################
 
-import sys, os, json
+def yaml_to_json(yaml_files, output_dir):
 
-# marbl_root is the top-level MARBL directory
-marbl_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-sys.path.append(marbl_root)
+    import json
+    # This tool requires PyYAML, error if it is not available
+    try:
+        import yaml
+    except:
+        logger.error("Can not find PyYAML library")
+        sys.exit(1)
+
+    # Routines to verify that YAML files meet MARBL formatting requirements
+    from MARBL_tools import settings_dictionary_is_consistent, diagnostics_dictionary_is_consistent
 
-# Parse command-line arguments (marbl_root is used to set default for YAML file location)
-args = _parse_args(marbl_root)
+    # Read YAML files
+    for rel_file in yaml_files:
+        filename = os.path.abspath(rel_file)
+        yaml_filename = filename.split(os.path.sep)[-1]
+        if not os.path.isfile(filename):
+            logger.error("File not found: "+filename)
+            sys.exit(1)
+        with open(filename) as file_in:
+            yaml_in = yaml.safe_load(file_in)
+
+        # YAML consistency checks (MARBL expects these files to be formatted
+        # in a specific way)
+        if yaml_filename.startswith("settings"):
+            check_func=settings_dictionary_is_consistent
+        elif yaml_filename.startswith("diagnostics"):
+            check_func=diagnostics_dictionary_is_consistent
+        else:
+            logger.error("Can not find consistency check for %s" % filename)
+            sys.exit(1)
 
-# Set up logging
-import logging
-logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
-logger = logging.getLogger("__name__")
+        logger.info("Running consistency check for %s" % filename)
+        if not check_func(yaml_in):
+            logger.error("Formatting error in %s reported from %s" % (filename, check_func.__name__))
+            sys.exit(1)
 
-# This tool requires PyYAML, error if it is not available
-try:
-    import yaml
-except:
-    logger.error("Can not find PyYAML library")
-    sys.exit(1)
+        # Write JSON file
+        if yaml_filename[-5:].lower() == ".yaml":
+            json_filename = yaml_filename[:-5] + ".json"
+        elif yaml_filename[-4:].lower() == ".yml":
+            json_filename = yaml_filename[:-4] + ".json"
+        else:
+            json_filename = yaml_filename + ".json"
+        json_fullname = os.path.abspath(os.path.join(output_dir, json_filename))
+        with open(json_fullname, "w") as file_out:
+            logger.info('Writing %s' % json_fullname)
+            json.dump(yaml_in, file_out, separators=(',', ': '), sort_keys=True, indent=3)
 
-# Routines to verify that YAML files meet MARBL formatting requirements
-from MARBL_tools import settings_dictionary_is_consistent, diagnostics_dictionary_is_consistent
+#######################################
 
-# Read YAML files
-for filename in [args.settings_file, args.diags_file]:
-    if not os.path.isfile(filename):
-        logger.error("File not found: "+filename)
-        sys.exit(1)
-    with open(filename) as file_in:
-        yaml_in = yaml.safe_load(file_in)
-
-    # YAML consistency checks (MARBL expects these files to be formatted
-    # in a specific way)
-    if filename == args.settings_file:
-        if not settings_dictionary_is_consistent(yaml_in):
-            logger.error("Formatting error in %s" % filename)
-            sys.exit(1)
-    if filename == args.diags_file:
-        if not diagnostics_dictionary_is_consistent(yaml_in):
-            logger.error("Formatting error in %s" % filename)
-            sys.exit(1)
+if __name__ == "__main__":
 
-    # Write JSON file
-    yaml_filename = filename.split(os.path.sep)[-1]
-    if yaml_filename[-4:].lower() == "yaml":
-        json_filename = yaml_filename[:-4] + "json"
-    elif yaml_filename[-3:].lower() == "yml":
-        json_filename = yaml_filename[:-3] + "json"
+    # marbl_root is the top-level MARBL directory, which is a level above the directory containing this script
+    # * Do some string manipulation to make marbl_root as human-readable as possible because it appears in the
+    #   output if you run this script with the "-h" option
+    script_dir = os.path.dirname(sys.argv[0])
+    if script_dir == '.':
+        marbl_root = '..'
+    elif script_dir.endswith('MARBL_tools'):
+        marbl_root = script_dir[:-12]
     else:
-        json_filename = yaml_filename + ".json"
-    with open(os.path.join(args.output_dir, json_filename), "w") as file_out:
-        json.dump(yaml_in, file_out, separators=(',', ': '), sort_keys=True, indent=3)
+        marbl_root = os.path.join(script_dir, '..')
+    yaml_dir = os.path.join(marbl_root, "defaults")
+    sys.path.append(marbl_root)
 
+    # Parse command-line arguments (marbl_root is used to set default for YAML file location)
+    args = _parse_args(marbl_root)
 
+    # Set up logging
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
+    logger = logging.getLogger("__name__")
 
+    yaml_to_json(args.yaml_files, args.output_dir)