From 4a0e6f8902fc8a988182a34b5070fe89a0920b9e Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Tue, 19 Sep 2017 16:19:29 +0100 Subject: [PATCH] int parameters can now be formatted by %d --- lib/cylc/cfgspec/suite.py | 4 +- lib/cylc/config.py | 15 ++--- lib/cylc/param_expand.py | 86 +++++++++++++----------- tests/param_expand/01-basic.t | 50 +++++++++++--- tests/param_expand/01-basic/10.graph.ref | 17 +++++ tests/param_expand/01-basic/11.graph.ref | 17 +++++ 6 files changed, 126 insertions(+), 63 deletions(-) create mode 100644 tests/param_expand/01-basic/10.graph.ref create mode 100644 tests/param_expand/01-basic/11.graph.ref diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py index 350526445db..7aa4119f4f1 100644 --- a/lib/cylc/cfgspec/suite.py +++ b/lib/cylc/cfgspec/suite.py @@ -177,9 +177,7 @@ def _coerce_parameter_list(value, keys, _): not str(item).isdigit() for item in items): return items else: - items = [int(item) for item in items] - n_digits = len(str(max(items))) - return [str(item).zfill(n_digits) for item in sorted(items)] + return [int(item) for item in items] coercers['cycletime'] = _coerce_cycletime coercers['cycletime_format'] = _coerce_cycletime_format diff --git a/lib/cylc/config.py b/lib/cylc/config.py index e0477cd7a93..0d07591cf92 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -248,16 +248,13 @@ def __init__(self, suite, fpath, template_vars=None, # Set default parameter expansion templates if necessary. for pname, pvalues in parameter_values.items(): - if pname not in parameter_templates: - try: - [int(i) for i in pvalues] - except ValueError: - # Don't prefix string values with the parameter name. - parameter_templates[pname] = "_%(" + pname + ")s" + if pvalues and pname not in parameter_templates: + if all(isinstance(pvalue, int) for pvalue in pvalues): + parameter_templates[pname] = r'_%s%%(%s)0%dd' % ( + pname, pname, len(str(max(pvalues)))) else: - # All int values, prefix values with the parameter name. - parameter_templates[pname] = ( - "_" + pname + "%(" + pname + ")s") + # Don't prefix string values with the parameter name. + parameter_templates[pname] = r'_%%(%s)s' % pname # Expand parameters in 'special task' lists. if 'special tasks' in self.cfg['scheduling']: diff --git a/lib/cylc/param_expand.py b/lib/cylc/param_expand.py index dd52463f674..1cde0185299 100755 --- a/lib/cylc/param_expand.py +++ b/lib/cylc/param_expand.py @@ -154,14 +154,11 @@ def expand(self, runtime_heading): elif sval.startswith('='): # Check that specific parameter values exist. val = sval[1:].strip() - # Pad integer values here. try: - int(val) + nval = int(val) except ValueError: nval = val - else: - nval = val.zfill(len(self.param_cfg[pname][0])) - if not item_in_iterable(nval, self.param_cfg[pname]): + if not item_in_iterable(val, self.param_cfg[pname]): raise ParamExpandError( "ERROR, parameter %s out of range: %s" % ( pname, p_tmpl)) @@ -236,6 +233,9 @@ def replace_params(self, name_in, param_values, origin): class GraphExpander(object): """Handle parameter expansion of graph string lines.""" + _REMOVE = -32768 + _REMOVE_REC = re.compile(r'^.*' + str(_REMOVE) + r'.*?=>\s*?') + def __init__(self, parameters): """Initialize the parameterized task name expander. @@ -296,7 +296,8 @@ def expand(self, line): except ValueError: nval = val else: - nval = val.zfill(len(self.param_cfg[pname][0])) + nval = val.zfill( + len(str(self.param_cfg[pname][0]))) if nval != val: line = re.sub(item, '%s=%s' % (pname, nval), line) @@ -322,6 +323,7 @@ def _expand_graph(self, line, all_params, values = {} if not param_list: # Inner loop. + import sys for p_group in set(REC_P_GROUP.findall(line)): # Parameters must be expanded in the order found. param_values = OrderedDictWithDefaults() @@ -332,14 +334,18 @@ def _expand_graph(self, line, all_params, param_values[pname] = values[pname] elif offs.startswith('='): # Specific value. - param_values[pname] = offs[1:] + try: + # Template may require an integer + param_values[pname] = int(offs[1:]) + except ValueError: + param_values[pname] = offs[1:] else: # Index offset. plist = all_params[pname] cur_idx = plist.index(values[pname]) off_idx = cur_idx + int(offs) if off_idx < 0: - offval = "----" + offval = self._REMOVE else: offval = plist[off_idx] param_values[pname] = offval @@ -352,7 +358,7 @@ def _expand_graph(self, line, all_params, 'defined.' % str(exc.args[0])) line = re.sub('<' + p_group + '>', repl, line) # Remove out-of-range nodes to first arrow. - line = re.sub('^.*----.*?=>\s*?', '', line) + line = self._REMOVE_REC.sub('', line) line_set.add(line) else: # Recurse through index ranges. @@ -367,9 +373,9 @@ class TestParamExpand(unittest.TestCase): def setUp(self): """Create some parameters and templates for use in tests.""" - ivals = [str(i) for i in range(2)] - jvals = [str(j) for j in range(3)] - kvals = [str(k) for k in range(2)] + ivals = [i for i in range(2)] + jvals = [j for j in range(3)] + kvals = [k for k in range(2)] params_map = {'i': ivals, 'j': jvals, 'k': kvals} templates = {'i': '_i%(i)s', 'j': '_j%(j)s', @@ -381,56 +387,56 @@ def test_name_one_param(self): """Test name expansion and returned value for a single parameter.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_j0', {'j': '0'}), - ('foo_j1', {'j': '1'}), - ('foo_j2', {'j': '2'})] + [('foo_j0', {'j': 0}), + ('foo_j1', {'j': 1}), + ('foo_j2', {'j': 2})] ) def test_name_two_params(self): """Test name expansion and returned values for two parameters.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j0', {'i': '0', 'j': '0'}), - ('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i0_j2', {'i': '0', 'j': '2'}), - ('foo_i1_j0', {'i': '1', 'j': '0'}), - ('foo_i1_j1', {'i': '1', 'j': '1'}), - ('foo_i1_j2', {'i': '1', 'j': '2'})] + [('foo_i0_j0', {'i': 0, 'j': 0}), + ('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i0_j2', {'i': 0, 'j': 2}), + ('foo_i1_j0', {'i': 1, 'j': 0}), + ('foo_i1_j1', {'i': 1, 'j': 1}), + ('foo_i1_j2', {'i': 1, 'j': 2})] ) def test_name_two_names(self): """Test name expansion for two names.""" self.assertEqual( self.name_expander.expand('foo, bar'), - [('foo_i0', {'i': '0'}), - ('foo_i1', {'i': '1'}), - ('bar_j0', {'j': '0'}), - ('bar_j1', {'j': '1'}), - ('bar_j2', {'j': '2'})] + [('foo_i0', {'i': 0}), + ('foo_i1', {'i': 1}), + ('bar_j0', {'j': 0}), + ('bar_j1', {'j': 1}), + ('bar_j2', {'j': 2})] ) def test_name_specific_val_1(self): """Test singling out a specific value, in name expansion.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0', {'i': '0'})] + [('foo_i0', {'i': 0})] ) def test_name_specific_val_2(self): """Test specific value in the first parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j0', {'i': '0', 'j': '0'}), - ('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i0_j2', {'i': '0', 'j': '2'})] + [('foo_i0_j0', {'i': 0, 'j': 0}), + ('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i0_j2', {'i': 0, 'j': 2})] ) def test_name_specific_val_3(self): """Test specific value in the second parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i1_j1', {'i': '1', 'j': '1'})] + [('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i1_j1', {'i': 1, 'j': 1})] ) def test_name_fail_bare_value(self): @@ -458,14 +464,14 @@ def test_name_multiple(self): """Test expansion of two names, with one and two parameters.""" self.assertEqual( self.name_expander.expand('foo, bar'), - [('foo_i0', {'i': '0'}), - ('foo_i1', {'i': '1'}), - ('bar_i0_j0', {'i': '0', 'j': '0'}), - ('bar_i0_j1', {'i': '0', 'j': '1'}), - ('bar_i0_j2', {'i': '0', 'j': '2'}), - ('bar_i1_j0', {'i': '1', 'j': '0'}), - ('bar_i1_j1', {'i': '1', 'j': '1'}), - ('bar_i1_j2', {'i': '1', 'j': '2'})] + [('foo_i0', {'i': 0}), + ('foo_i1', {'i': 1}), + ('bar_i0_j0', {'i': 0, 'j': 0}), + ('bar_i0_j1', {'i': 0, 'j': 1}), + ('bar_i0_j2', {'i': 0, 'j': 2}), + ('bar_i1_j0', {'i': 1, 'j': 0}), + ('bar_i1_j1', {'i': 1, 'j': 1}), + ('bar_i1_j2', {'i': 1, 'j': 2})] ) def test_graph_expand_1(self): diff --git a/tests/param_expand/01-basic.t b/tests/param_expand/01-basic.t index 23e37c4cc49..118f3e0147b 100644 --- a/tests/param_expand/01-basic.t +++ b/tests/param_expand/01-basic.t @@ -17,7 +17,7 @@ #------------------------------------------------------------------------------- # Check tasks and graph generated by parameter expansion. . "$(dirname "$0")/test_header" -set_test_number 20 +set_test_number 24 cat >'suite.rc' <<'__SUITE__' [cylc] @@ -39,7 +39,6 @@ qux => waz [[qux]] [[waz]] __SUITE__ - run_ok "${TEST_NAME_BASE}-1" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'1.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/1.graph.ref" '1.graph' @@ -59,7 +58,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-2" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'2.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/2.graph.ref" '2.graph' @@ -79,7 +77,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-3" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'3.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/3.graph.ref" '3.graph' @@ -99,7 +96,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-4" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'4.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/4.graph.ref" '4.graph' @@ -119,7 +115,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-5" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-5.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = space is dangerous: space is dangerous: bad value @@ -140,7 +135,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-6" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-6.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = mix, 1..10: mixing int range and str @@ -161,7 +155,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-7" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'7.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/7.graph.ref" '7.graph' @@ -181,7 +174,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-8" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-8.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = 1..2 3..4: 1..2 3..4: bad value @@ -202,7 +194,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-9" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-9.stderr" <<'__ERR__' ERROR, parameter i is not defined in foo @@ -220,10 +211,47 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-9" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-9.stderr" <<'__ERR__' ERROR, parameter i is not defined in : foo=>bar __ERR__ +cat >'suite.rc' <<'__SUITE__' +[cylc] + [[parameters]] + j = 1..5 + [[parameter templates]] + j = @%(j)03d +[scheduling] + [[dependencies]] + graph = "foo => bar" +[runtime] + [[root]] + script = true + [[foo]] + [[bar]] +__SUITE__ +run_ok "${TEST_NAME_BASE}-10" cylc validate "suite.rc" +cylc graph --reference 'suite.rc' >'10.graph' +cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/10.graph.ref" '10.graph' + +cat >'suite.rc' <<'__SUITE__' +[cylc] + [[parameters]] + j = 1..5 + [[parameter templates]] + j = +%%j%(j)03d +[scheduling] + [[dependencies]] + graph = "foo => bar" +[runtime] + [[root]] + script = true + [[foo]] + [[bar]] +__SUITE__ +run_ok "${TEST_NAME_BASE}-11" cylc validate "suite.rc" +cylc graph --reference 'suite.rc' >'11.graph' +cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/11.graph.ref" '11.graph' + exit diff --git a/tests/param_expand/01-basic/10.graph.ref b/tests/param_expand/01-basic/10.graph.ref new file mode 100644 index 00000000000..24b862c57a1 --- /dev/null +++ b/tests/param_expand/01-basic/10.graph.ref @@ -0,0 +1,17 @@ +edge "foo@001.1" "bar@001.1" solid +edge "foo@002.1" "bar@002.1" solid +edge "foo@003.1" "bar@003.1" solid +edge "foo@004.1" "bar@004.1" solid +edge "foo@005.1" "bar@005.1" solid +graph +node "bar@001.1" "bar@001\n1" unfilled ellipse black +node "bar@002.1" "bar@002\n1" unfilled ellipse black +node "bar@003.1" "bar@003\n1" unfilled ellipse black +node "bar@004.1" "bar@004\n1" unfilled ellipse black +node "bar@005.1" "bar@005\n1" unfilled ellipse black +node "foo@001.1" "foo@001\n1" unfilled ellipse black +node "foo@002.1" "foo@002\n1" unfilled ellipse black +node "foo@003.1" "foo@003\n1" unfilled ellipse black +node "foo@004.1" "foo@004\n1" unfilled ellipse black +node "foo@005.1" "foo@005\n1" unfilled ellipse black +stop diff --git a/tests/param_expand/01-basic/11.graph.ref b/tests/param_expand/01-basic/11.graph.ref new file mode 100644 index 00000000000..3cf6addbf8b --- /dev/null +++ b/tests/param_expand/01-basic/11.graph.ref @@ -0,0 +1,17 @@ +edge "foo+%j001.1" "bar+%j001.1" solid +edge "foo+%j002.1" "bar+%j002.1" solid +edge "foo+%j003.1" "bar+%j003.1" solid +edge "foo+%j004.1" "bar+%j004.1" solid +edge "foo+%j005.1" "bar+%j005.1" solid +graph +node "bar+%j001.1" "bar+%j001\n1" unfilled ellipse black +node "bar+%j002.1" "bar+%j002\n1" unfilled ellipse black +node "bar+%j003.1" "bar+%j003\n1" unfilled ellipse black +node "bar+%j004.1" "bar+%j004\n1" unfilled ellipse black +node "bar+%j005.1" "bar+%j005\n1" unfilled ellipse black +node "foo+%j001.1" "foo+%j001\n1" unfilled ellipse black +node "foo+%j002.1" "foo+%j002\n1" unfilled ellipse black +node "foo+%j003.1" "foo+%j003\n1" unfilled ellipse black +node "foo+%j004.1" "foo+%j004\n1" unfilled ellipse black +node "foo+%j005.1" "foo+%j005\n1" unfilled ellipse black +stop