diff --git a/doc/src/cylc-user-guide/cug.tex b/doc/src/cylc-user-guide/cug.tex index f4839557f8e..97758019a4b 100644 --- a/doc/src/cylc-user-guide/cug.tex +++ b/doc/src/cylc-user-guide/cug.tex @@ -4498,12 +4498,28 @@ \subsubsection{Parameter Expansion} \begin{lstlisting} [cylc] [[parameters]] + # parameters: "ship", "buoy", "plane" + # default task suffixes: _ship, _buoy, _plane obs = ship, buoy, plane - run = 1..5 # 1, 2, 3, 4, 5 - idx = 1..9..2 # 1, 3, 5, 7, 9 - i = 1..5..2, 10, 11..13 # 1, 3, 5, 10, 11, 12, 13 - item = 0, 1, e, pi, i # "0", "1", "e", "pi", "i" - p = one, two, 3..5 # ERROR + + # parameters: 1, 2, 3, 4, 5 + # default task suffixes: _run1, _run2, _run3, _run4, _run5 + run = 1..5 + + # parameters: 1, 3, 5, 7, 9 + # default task suffixes: _idx1, _idx3, _idx5, _idx7, _idx9 + idx = 1..9..2 + + # parameters: 1, 3, 5, 10, 11, 12, 13 + # default task suffixes: _i01, _i03, _i05, _i10, _i11, _i12, _i13 + i = 1..5..2, 10, 11..13 + + # parameters: "0", "1", "e", "pi", "i" + # default task suffixes: _0, _1, _e, _pi, _i + item = 0, 1, e, pi, i + + # ERROR: mix strings with int range + p = one, two, 3..5 \end{lstlisting} Then angle brackets denote use of these parameters throughout the suite definition. For the values above, this parameterized name: @@ -4604,13 +4620,24 @@ \subsubsection{Parameter Expansion} \paragraph{Zero-Padded Integer Values} -Integer parameter values are zero-padded according to the size of their -largest value, so \lstinline@foo

@ for \lstinline@p = 9..10@ expands to +Integer parameter values are given a default template for generating task +suffixes that are zero-padded according to the size of their largest value. +For example, the default template for \lstinline@p = 9..10@ would be +\lstinline@_p%(p)02d@, so that \lstinline@foo

@ would become \lstinline@foo_p09, foo_p10@. -To get thicker padding, prepend extra zeroes to the upper range value: -\lstinline@foo

@ for \lstinline@p = 9..010@ expands to -\lstinline@foo_p009, foo_p010@. +To get thicker padding and/or alternate suffixes, use a template. E.g.: + +\begin{lstlisting} +[cylc] + [[parameters]] + i = 1..9 + p = 3..14 + [[parameter templates]] + i = _i%(i)02d # suffixes = _i01, _i02, ..., _i09 + # A double-percent gives a literal percent character + p = %%p%(p)03d # suffixes = %p003, %p004, ..., %p013, %p014 +\end{lstlisting} \subsubsection{Passing Parameter Values To Tasks} @@ -4658,11 +4685,11 @@ \subsubsection{Selecting Partial Parameter Ranges} \begin{lstlisting} [cylc] [[parameters]] - run = 1..10 # 01, 02, ..., 10 - runx = 1..03 # 01, 02, 03 (note `03' to get correct padding) + run = 1..10 # 1, 2, ..., 10 + runx = 1..3 # 1, 2, 3 [[parameter templates]] - run = _R%(run)s - runx = _R%(runx)s + run = _R%(run)02d # _R01, _R02, ..., _R10 + runx = _R%(runx)02d # _R01, _R02, _R03 [scheduling] [[dependencies]] graph = """model => post diff --git a/doc/src/cylc-user-guide/suiterc.tex b/doc/src/cylc-user-guide/suiterc.tex index 59f952b4bf1..15b6b0a415b 100644 --- a/doc/src/cylc-user-guide/suiterc.tex +++ b/doc/src/cylc-user-guide/suiterc.tex @@ -295,11 +295,12 @@ \subsection{[cylc]} \begin{myitemize} \item {\em type:} a Python-style string template - \item {\em default} for an integer-valued parameter \lstinline=p=: - \lstinline=_p%(p)s= \\ + \item {\em default} for integer parameters \lstinline=p=: + \lstinline=_p%(p)0Nd= \\ + where N is the number of digits of the maximum integer value, e.g.\ \lstinline=foo= becomes \lstinline=foo_run3= for \lstinline@run@ value \lstinline@3@. - \item {\em default} for a non integer-valued parameter \lstinline=p=: + \item {\em default} for non-integer parameters \lstinline=p=: \lstinline=_%(p)s= \\ e.g.\ \lstinline=foo= becomes \lstinline=foo_top= for \lstinline@run@ value \lstinline@top@. @@ -309,10 +310,8 @@ \subsection{[cylc]} \end{myitemize} Note that the values of a parameter named \lstinline=p= are substituted for -\lstinline=%(p)s=. The \lstinline=s= indicates a string value and should -be used even for integer-valued parameters, because cylc converts integer -parameter values to strings, with (depending on size) zero-padding. In -\lstinline=_run%(run)s= the first ``run'' is a string literal, and the second +\lstinline=%(p)s=. +In \lstinline=_run%(run)s= the first ``run'' is a string literal, and the second gets substituted with each value of the parameter. \subsubsection[{[[}events{]]}]{[cylc] \textrightarrow [[events]]} diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py index 350526445db..7aa4119f4f1 100644 --- a/lib/cylc/cfgspec/suite.py +++ b/lib/cylc/cfgspec/suite.py @@ -177,9 +177,7 @@ def _coerce_parameter_list(value, keys, _): not str(item).isdigit() for item in items): return items else: - items = [int(item) for item in items] - n_digits = len(str(max(items))) - return [str(item).zfill(n_digits) for item in sorted(items)] + return [int(item) for item in items] coercers['cycletime'] = _coerce_cycletime coercers['cycletime_format'] = _coerce_cycletime_format diff --git a/lib/cylc/config.py b/lib/cylc/config.py index e0477cd7a93..0d07591cf92 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -248,16 +248,13 @@ def __init__(self, suite, fpath, template_vars=None, # Set default parameter expansion templates if necessary. for pname, pvalues in parameter_values.items(): - if pname not in parameter_templates: - try: - [int(i) for i in pvalues] - except ValueError: - # Don't prefix string values with the parameter name. - parameter_templates[pname] = "_%(" + pname + ")s" + if pvalues and pname not in parameter_templates: + if all(isinstance(pvalue, int) for pvalue in pvalues): + parameter_templates[pname] = r'_%s%%(%s)0%dd' % ( + pname, pname, len(str(max(pvalues)))) else: - # All int values, prefix values with the parameter name. - parameter_templates[pname] = ( - "_" + pname + "%(" + pname + ")s") + # Don't prefix string values with the parameter name. + parameter_templates[pname] = r'_%%(%s)s' % pname # Expand parameters in 'special task' lists. if 'special tasks' in self.cfg['scheduling']: diff --git a/lib/cylc/param_expand.py b/lib/cylc/param_expand.py index dd52463f674..05ff1eeb9bb 100755 --- a/lib/cylc/param_expand.py +++ b/lib/cylc/param_expand.py @@ -15,13 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -import re -import unittest -from copy import copy -from task_id import TaskID -from parsec.OrderedDict import OrderedDictWithDefaults - """Parameter expansion for runtime namespace names and graph strings. Uses recursion to achieve nested looping over any number of parameters. In its @@ -65,6 +58,12 @@ def expand(template, params, results, values=None): #------------------------------------------------------------------------------ """ +import re +import unittest + +from cylc.task_id import TaskID +from parsec.OrderedDict import OrderedDictWithDefaults + # To split runtime heading name lists. REC_NAMES = re.compile(r'(?:[^,<]|\<[^>]*\>)+') # To extract 'name', '', and 'other' from @@ -154,13 +153,10 @@ def expand(self, runtime_heading): elif sval.startswith('='): # Check that specific parameter values exist. val = sval[1:].strip() - # Pad integer values here. try: - int(val) + nval = int(val) except ValueError: nval = val - else: - nval = val.zfill(len(self.param_cfg[pname][0])) if not item_in_iterable(nval, self.param_cfg[pname]): raise ParamExpandError( "ERROR, parameter %s out of range: %s" % ( @@ -196,7 +192,7 @@ def _expand_name(self, str_tmpl, param_list, results, spec_vals=None): spec_vals = {} if not param_list: # Inner loop. - current_values = copy(spec_vals) + current_values = dict(spec_vals) try: results.append((str_tmpl % current_values, current_values)) except KeyError as exc: @@ -236,6 +232,9 @@ def replace_params(self, name_in, param_values, origin): class GraphExpander(object): """Handle parameter expansion of graph string lines.""" + _REMOVE = -32768 + _REMOVE_REC = re.compile(r'^.*' + str(_REMOVE) + r'.*?=>\s*?') + def __init__(self, parameters): """Initialize the parameterized task name expander. @@ -290,16 +289,10 @@ def expand(self, line): elif offs.startswith('='): # Check that specific parameter values exist. val = offs[1:] - # Pad integer values here. try: - int(val) + nval = int(val) except ValueError: nval = val - else: - nval = val.zfill(len(self.param_cfg[pname][0])) - if nval != val: - line = re.sub(item, - '%s=%s' % (pname, nval), line) if not item_in_iterable(nval, self.param_cfg[pname]): raise ParamExpandError( "ERROR, parameter %s out of range: %s" % ( @@ -332,14 +325,18 @@ def _expand_graph(self, line, all_params, param_values[pname] = values[pname] elif offs.startswith('='): # Specific value. - param_values[pname] = offs[1:] + try: + # Template may require an integer + param_values[pname] = int(offs[1:]) + except ValueError: + param_values[pname] = offs[1:] else: # Index offset. plist = all_params[pname] cur_idx = plist.index(values[pname]) off_idx = cur_idx + int(offs) if off_idx < 0: - offval = "----" + offval = self._REMOVE else: offval = plist[off_idx] param_values[pname] = offval @@ -352,7 +349,7 @@ def _expand_graph(self, line, all_params, 'defined.' % str(exc.args[0])) line = re.sub('<' + p_group + '>', repl, line) # Remove out-of-range nodes to first arrow. - line = re.sub('^.*----.*?=>\s*?', '', line) + line = self._REMOVE_REC.sub('', line) line_set.add(line) else: # Recurse through index ranges. @@ -367,9 +364,9 @@ class TestParamExpand(unittest.TestCase): def setUp(self): """Create some parameters and templates for use in tests.""" - ivals = [str(i) for i in range(2)] - jvals = [str(j) for j in range(3)] - kvals = [str(k) for k in range(2)] + ivals = list(range(2)) + jvals = list(range(3)) + kvals = list(range(2)) params_map = {'i': ivals, 'j': jvals, 'k': kvals} templates = {'i': '_i%(i)s', 'j': '_j%(j)s', @@ -381,56 +378,56 @@ def test_name_one_param(self): """Test name expansion and returned value for a single parameter.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_j0', {'j': '0'}), - ('foo_j1', {'j': '1'}), - ('foo_j2', {'j': '2'})] + [('foo_j0', {'j': 0}), + ('foo_j1', {'j': 1}), + ('foo_j2', {'j': 2})] ) def test_name_two_params(self): """Test name expansion and returned values for two parameters.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j0', {'i': '0', 'j': '0'}), - ('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i0_j2', {'i': '0', 'j': '2'}), - ('foo_i1_j0', {'i': '1', 'j': '0'}), - ('foo_i1_j1', {'i': '1', 'j': '1'}), - ('foo_i1_j2', {'i': '1', 'j': '2'})] + [('foo_i0_j0', {'i': 0, 'j': 0}), + ('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i0_j2', {'i': 0, 'j': 2}), + ('foo_i1_j0', {'i': 1, 'j': 0}), + ('foo_i1_j1', {'i': 1, 'j': 1}), + ('foo_i1_j2', {'i': 1, 'j': 2})] ) def test_name_two_names(self): """Test name expansion for two names.""" self.assertEqual( self.name_expander.expand('foo, bar'), - [('foo_i0', {'i': '0'}), - ('foo_i1', {'i': '1'}), - ('bar_j0', {'j': '0'}), - ('bar_j1', {'j': '1'}), - ('bar_j2', {'j': '2'})] + [('foo_i0', {'i': 0}), + ('foo_i1', {'i': 1}), + ('bar_j0', {'j': 0}), + ('bar_j1', {'j': 1}), + ('bar_j2', {'j': 2})] ) def test_name_specific_val_1(self): """Test singling out a specific value, in name expansion.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0', {'i': '0'})] + [('foo_i0', {'i': 0})] ) def test_name_specific_val_2(self): """Test specific value in the first parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j0', {'i': '0', 'j': '0'}), - ('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i0_j2', {'i': '0', 'j': '2'})] + [('foo_i0_j0', {'i': 0, 'j': 0}), + ('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i0_j2', {'i': 0, 'j': 2})] ) def test_name_specific_val_3(self): """Test specific value in the second parameter of a pair.""" self.assertEqual( self.name_expander.expand('foo'), - [('foo_i0_j1', {'i': '0', 'j': '1'}), - ('foo_i1_j1', {'i': '1', 'j': '1'})] + [('foo_i0_j1', {'i': 0, 'j': 1}), + ('foo_i1_j1', {'i': 1, 'j': 1})] ) def test_name_fail_bare_value(self): @@ -458,14 +455,14 @@ def test_name_multiple(self): """Test expansion of two names, with one and two parameters.""" self.assertEqual( self.name_expander.expand('foo, bar'), - [('foo_i0', {'i': '0'}), - ('foo_i1', {'i': '1'}), - ('bar_i0_j0', {'i': '0', 'j': '0'}), - ('bar_i0_j1', {'i': '0', 'j': '1'}), - ('bar_i0_j2', {'i': '0', 'j': '2'}), - ('bar_i1_j0', {'i': '1', 'j': '0'}), - ('bar_i1_j1', {'i': '1', 'j': '1'}), - ('bar_i1_j2', {'i': '1', 'j': '2'})] + [('foo_i0', {'i': 0}), + ('foo_i1', {'i': 1}), + ('bar_i0_j0', {'i': 0, 'j': 0}), + ('bar_i0_j1', {'i': 0, 'j': 1}), + ('bar_i0_j2', {'i': 0, 'j': 2}), + ('bar_i1_j0', {'i': 1, 'j': 0}), + ('bar_i1_j1', {'i': 1, 'j': 1}), + ('bar_i1_j2', {'i': 1, 'j': 2})] ) def test_graph_expand_1(self): diff --git a/tests/param_expand/01-basic.t b/tests/param_expand/01-basic.t index 23e37c4cc49..118f3e0147b 100644 --- a/tests/param_expand/01-basic.t +++ b/tests/param_expand/01-basic.t @@ -17,7 +17,7 @@ #------------------------------------------------------------------------------- # Check tasks and graph generated by parameter expansion. . "$(dirname "$0")/test_header" -set_test_number 20 +set_test_number 24 cat >'suite.rc' <<'__SUITE__' [cylc] @@ -39,7 +39,6 @@ qux => waz [[qux]] [[waz]] __SUITE__ - run_ok "${TEST_NAME_BASE}-1" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'1.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/1.graph.ref" '1.graph' @@ -59,7 +58,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-2" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'2.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/2.graph.ref" '2.graph' @@ -79,7 +77,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-3" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'3.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/3.graph.ref" '3.graph' @@ -99,7 +96,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-4" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'4.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/4.graph.ref" '4.graph' @@ -119,7 +115,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-5" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-5.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = space is dangerous: space is dangerous: bad value @@ -140,7 +135,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-6" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-6.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = mix, 1..10: mixing int range and str @@ -161,7 +155,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_ok "${TEST_NAME_BASE}-7" cylc validate "suite.rc" cylc graph --reference 'suite.rc' >'7.graph' cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/7.graph.ref" '7.graph' @@ -181,7 +174,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-8" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-8.stderr" <<'__ERR__' Illegal parameter value: [cylc][parameters]i = 1..2 3..4: 1..2 3..4: bad value @@ -202,7 +194,6 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-9" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-9.stderr" <<'__ERR__' ERROR, parameter i is not defined in foo @@ -220,10 +211,47 @@ foo => bar [[foo]] [[bar]] __SUITE__ - run_fail "${TEST_NAME_BASE}-9" cylc validate "suite.rc" cmp_ok "${TEST_NAME_BASE}-9.stderr" <<'__ERR__' ERROR, parameter i is not defined in : foo=>bar __ERR__ +cat >'suite.rc' <<'__SUITE__' +[cylc] + [[parameters]] + j = 1..5 + [[parameter templates]] + j = @%(j)03d +[scheduling] + [[dependencies]] + graph = "foo => bar" +[runtime] + [[root]] + script = true + [[foo]] + [[bar]] +__SUITE__ +run_ok "${TEST_NAME_BASE}-10" cylc validate "suite.rc" +cylc graph --reference 'suite.rc' >'10.graph' +cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/10.graph.ref" '10.graph' + +cat >'suite.rc' <<'__SUITE__' +[cylc] + [[parameters]] + j = 1..5 + [[parameter templates]] + j = +%%j%(j)03d +[scheduling] + [[dependencies]] + graph = "foo => bar" +[runtime] + [[root]] + script = true + [[foo]] + [[bar]] +__SUITE__ +run_ok "${TEST_NAME_BASE}-11" cylc validate "suite.rc" +cylc graph --reference 'suite.rc' >'11.graph' +cmp_ok "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/11.graph.ref" '11.graph' + exit diff --git a/tests/param_expand/01-basic/10.graph.ref b/tests/param_expand/01-basic/10.graph.ref new file mode 100644 index 00000000000..24b862c57a1 --- /dev/null +++ b/tests/param_expand/01-basic/10.graph.ref @@ -0,0 +1,17 @@ +edge "foo@001.1" "bar@001.1" solid +edge "foo@002.1" "bar@002.1" solid +edge "foo@003.1" "bar@003.1" solid +edge "foo@004.1" "bar@004.1" solid +edge "foo@005.1" "bar@005.1" solid +graph +node "bar@001.1" "bar@001\n1" unfilled ellipse black +node "bar@002.1" "bar@002\n1" unfilled ellipse black +node "bar@003.1" "bar@003\n1" unfilled ellipse black +node "bar@004.1" "bar@004\n1" unfilled ellipse black +node "bar@005.1" "bar@005\n1" unfilled ellipse black +node "foo@001.1" "foo@001\n1" unfilled ellipse black +node "foo@002.1" "foo@002\n1" unfilled ellipse black +node "foo@003.1" "foo@003\n1" unfilled ellipse black +node "foo@004.1" "foo@004\n1" unfilled ellipse black +node "foo@005.1" "foo@005\n1" unfilled ellipse black +stop diff --git a/tests/param_expand/01-basic/11.graph.ref b/tests/param_expand/01-basic/11.graph.ref new file mode 100644 index 00000000000..3cf6addbf8b --- /dev/null +++ b/tests/param_expand/01-basic/11.graph.ref @@ -0,0 +1,17 @@ +edge "foo+%j001.1" "bar+%j001.1" solid +edge "foo+%j002.1" "bar+%j002.1" solid +edge "foo+%j003.1" "bar+%j003.1" solid +edge "foo+%j004.1" "bar+%j004.1" solid +edge "foo+%j005.1" "bar+%j005.1" solid +graph +node "bar+%j001.1" "bar+%j001\n1" unfilled ellipse black +node "bar+%j002.1" "bar+%j002\n1" unfilled ellipse black +node "bar+%j003.1" "bar+%j003\n1" unfilled ellipse black +node "bar+%j004.1" "bar+%j004\n1" unfilled ellipse black +node "bar+%j005.1" "bar+%j005\n1" unfilled ellipse black +node "foo+%j001.1" "foo+%j001\n1" unfilled ellipse black +node "foo+%j002.1" "foo+%j002\n1" unfilled ellipse black +node "foo+%j003.1" "foo+%j003\n1" unfilled ellipse black +node "foo+%j004.1" "foo+%j004\n1" unfilled ellipse black +node "foo+%j005.1" "foo+%j005\n1" unfilled ellipse black +stop