Broadcast: settings from file

bin/cylc-broadcast

@@ -48,12 +48,26 @@ member_n (there is no direct broadcast to member_n to cancel).
 
 To broadcast a variable to all tasks (quote items with internal spaces):
   % cylc broadcast -s "[environment]VERSE = the quick brown fox" REG
+To do the same with a file:
+  % cat >'broadcast.rc' <<'__RC__'
+  % [environment]
+  %     VERSE = the quick brown fox
+  % __RC__
+  % cylc broadcast -F 'broadcast.rc' REG
 To cancel the same broadcast:
   % cylc broadcast --cancel "[environment]VERSE" REG
+If -F FILE was used, the same file can be used to cancel the broadcast:
+  % cylc broadcast -G 'broadcast.rc' REG
 
-Use -d/--display to see active broadcasts. Multiple set or cancel
-options can be used on the same command line. Broadcast cannot change
-[runtime] inheritance.
+Use -d/--display to see active broadcasts. Multiple --cancel options or
+multiple --set and --set-file options can be used on the same command line.
+Multiple --set and --set-file options are cumulative.
+
+The --set-file=FILE option can be used when broadcasting multiple values, or
+when the value contains newline or other metacharacters. If FILE is "-", read
+from standard input.
+
+Broadcast cannot change [runtime] inheritance.
 
 See also 'cylc reload' - reload a modified suite definition at run time."""
 
@@ -64,8 +78,8 @@ if '--use-ssh' in sys.argv[1:]:
     if remrun().execute(force_required=True):
         sys.exit(0)
 
-import os
 import re
+from tempfile import NamedTemporaryFile
 
 import cylc.flags
 from cylc.broadcast_report import (
@@ -75,10 +89,15 @@ from cylc.network.suite_broadcast_client import BroadcastClient
 from cylc.print_tree import print_tree
 from cylc.task_id import TaskID
 from cylc.cfgspec.suite import SPEC, upg
+from parsec.config import config
 from parsec.validate import validate
 
 
+REC_ITEM = re.compile(r'^\[([^\]]*)\](.*)$')
+
+
 def get_padding(settings, level=0, padding=0):
+    """Return the left padding for displaying a setting."""
     level += 1
     for key, val in settings.items():
         tmp = level * 2 + len(key)
@@ -90,23 +109,69 @@ def get_padding(settings, level=0, padding=0):
 
 
 def get_rdict(left, right=None):
-    # left is [section]item, or just item
+    """Check+transform left=right into a nested dict.
+
+    left can be key, [key], [key1]key2, [key1][key2], [key1][key2]key3, etc.
+    """
+    if left == "inherit":
+        raise ValueError(
+            "ERROR: Inheritance cannot be changed by broadcast")
     rdict = {}
-    m = re.match('^\[(.*)\](.*)$', left)
-    if m:
-        # [sect]item = right
-        sect, var = m.groups()
-        if not var:
-            rdict = {sect.strip(): right}
+    cur_dict = rdict
+    tail = left
+    while tail:
+        match = REC_ITEM.match(tail)
+        if match:
+            sect, tail = match.groups()
+            if tail:
+                # [sect]... = right
+                cur_dict[sect.strip()] = {}
+                cur_dict = cur_dict[sect.strip()]
+            else:
+                # [sect] = right
+                cur_dict[sect.strip()] = right
         else:
-            rdict = {sect.strip(): {var.strip(): right}}
-    else:
-        # item = right
-        rdict = {left: right}
+            # item = right
+            cur_dict[tail.strip()] = right
+            tail = None
+    upg({'runtime': {'__MANY__': rdict}}, 'test')
+    validate(rdict, SPEC['runtime']['__MANY__'])
     return rdict
 
 
+def files_to_settings(settings, setting_files, cancel_mode=False):
+    """Parse setting files, and append to settings."""
+    cfg = config(SPEC['runtime']['__MANY__'])
+    for setting_file in setting_files:
+        if setting_file == '-':
+            with NamedTemporaryFile() as handle:
+                handle.write(sys.stdin.read())
+                handle.seek(0, 0)
+                cfg.loadcfg(handle.name)
+        else:
+            cfg.loadcfg(setting_file)
+    stack = [([], cfg.get(sparse=True))]
+    while stack:
+        keys, item = stack.pop()
+        if isinstance(item, dict):
+            for key, value in item.items():
+                stack.append((keys + [key], value))
+        else:
+            settings.append({})
+            cur_setting = settings[-1]
+            while keys:
+                key = keys.pop(0)
+                if keys:
+                    cur_setting[key] = {}
+                    cur_setting = cur_setting[key]
+                elif cancel_mode:
+                    cur_setting[key] = None
+                else:
+                    cur_setting[key] = item
+
+
 def main():
+    """CLI for "cylc broadcast"."""
     parser = COP(__doc__, comms=True)
 
     parser.add_option(
@@ -125,13 +190,23 @@ def main():
     parser.add_option(
         "-s", "--set", metavar="[SEC]ITEM=VALUE",
         help="A [runtime] config item and value to broadcast.",
-        action="append", dest="set", default=[])
+        action="append", dest="settings", default=[])
+
+    parser.add_option(
+        "-F", "--set-file", "--file", metavar="FILE",
+        help="File with config to broadcast. Can be used multiple times.",
+        action="append", dest="setting_files", default=[])
 
     parser.add_option(
         "-c", "--cancel", metavar="[SEC]ITEM",
         help="An item-specific broadcast to cancel.",
         action="append", dest="cancel", default=[])
 
+    parser.add_option(
+        "-G", "--cancel-file", metavar="FILE",
+        help="File with broadcasts to cancel. Can be used multiple times.",
+        action="append", dest="cancel_files", default=[])
+
     parser.add_option(
         "-C", "--clear",
         help="Cancel all broadcasts, or with -p/--point, "
@@ -169,20 +244,9 @@ def main():
              "the broadcast config structure in raw Python form.",
         action="store_true", default=False, dest="raw")
 
-    (options, args) = parser.parse_args()
+    options, args = parser.parse_args()
     suite = args[0]
 
-    debug = False
-    if cylc.flags.debug:
-        debug = True
-    else:
-        try:
-            # from task execution environment
-            if os.environ['CYLC_DEBUG'] == 'True':
-                debug = True
-        except KeyError:
-            pass
-
     pclient = BroadcastClient(
         suite, options.owner, options.host, options.port,
         options.comms_timeout, my_uuid=options.set_uuid,
@@ -191,7 +255,7 @@ def main():
     if options.show or options.showtask:
         if options.showtask:
             try:
-                name, point_string = TaskID.split(options.showtask)
+                TaskID.split(options.showtask)
             except ValueError:
                 parser.error("TASKID must be " + TaskID.SYNTAX)
         settings = pclient.broadcast('get', task_id=options.showtask)
@@ -228,20 +292,16 @@ def main():
     if not point_strings:
         point_strings = ["*"]
 
-    if options.cancel:
+    if options.cancel or options.cancel_files:
         settings = []
         for option_item in options.cancel:
             if "=" in option_item:
                 raise ValueError(
                     "ERROR: --cancel=[SEC]ITEM does not take a value")
             option_item = option_item.strip()
-            if option_item == "inherit":
-                raise ValueError(
-                    "ERROR: Inheritance cannot be changed by broadcast")
             setting = get_rdict(option_item)
-            upg({'runtime': {'__MANY__': setting}}, 'test')
-            validate(setting, SPEC['runtime']['__MANY__'])
             settings.append(setting)
+        files_to_settings(settings, options.cancel_files, options.cancel)
         modified_settings, bad_options = pclient.broadcast(
             'clear', point_strings=point_strings,
             namespaces=namespaces, cancel_settings=settings
@@ -251,20 +311,16 @@ def main():
                 modified_settings, is_cancel=True)
         sys.exit(get_broadcast_bad_options_report(bad_options))
 
-    if options.set:
+    if options.settings or options.setting_files:
         settings = []
-        for option_item in options.set:
+        for option_item in options.settings:
             if "=" not in option_item:
                 raise ValueError(
                     "ERROR: --set=[SEC]ITEM=VALUE requires a value")
             lhs, rhs = [s.strip() for s in option_item.split("=", 1)]
-            if lhs == "inherit":
-                raise ValueError(
-                    "ERROR: Inheritance cannot be changed by broadcast")
             setting = get_rdict(lhs, rhs)
-            upg({'runtime': {'__MANY__': setting}}, 'test')
-            validate(setting, SPEC['runtime']['__MANY__'])
             settings.append(setting)
+        files_to_settings(settings, options.setting_files)
         modified_settings, bad_options = pclient.broadcast(
             'put', point_strings=point_strings,
             namespaces=namespaces, settings=settings

lib/cylc/network/https/base_client.py

@@ -201,7 +201,7 @@ def _get_data_from_url_with_urllib2(self, url, json_data, method=None):
         req = urllib2.Request(url, json_data, json_headers)
 
         # This is an unpleasant monkey patch, but there isn't an alternative.
-        # urllib2 uses POST iff there is a data payload, but that is not the
+        # urllib2 uses POST if there is a data payload, but that is not the
         # correct criterion. The difference is basically that POST changes
         # server state and GET doesn't.
         req.get_method = lambda: method

tests/broadcast/00-simple/suite.rc

@@ -40,7 +40,7 @@ set +x
 {
     # broadcast to all cycles and namespaces:
         cylc broadcast -s "[environment]BCAST = ROOT" $CYLC_SUITE_NAME
-    # broadcast to foo%20100808T00:
+    # broadcast to foo.20100808T00:
         cylc broadcast -p 20100808T00 -n foo -s "[environment]BCAST = FOO" $CYLC_SUITE_NAME
     # broadcast to bar at all cycles:
         cylc broadcast -n bar -s "[environment]BCAST = BAR" $CYLC_SUITE_NAME

tests/broadcast/10-file-1.t

@@ -0,0 +1,28 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2017 NIWA
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test broadcast settings in a file
+. "$(dirname "$0")/test_header"
+set_test_number 2
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
+suite_run_ok "${TEST_NAME_BASE}" \
+    cylc run --debug --reference-test "${SUITE_NAME}"
+
+purge_suite "${SUITE_NAME}"
+exit

tests/broadcast/10-file-1/broadcast.rc

@@ -0,0 +1,7 @@
+script=printenv CYLC_FOOBAR
+[environment]
+    CYLC_FOOBAR="""
+foo
+bar
+baz
+"""

tests/broadcast/10-file-1/reference.log

@@ -0,0 +1,4 @@
+2017-01-12T14:46:57Z INFO - Initial point: 1
+2017-01-12T14:46:57Z INFO - Final point: 1
+2017-01-12T14:46:57Z INFO - [t1.1] -triggered off []
+2017-01-12T14:47:00Z INFO - [t2.1] -triggered off ['t1.1']

tests/broadcast/10-file-1/suite.rc

@@ -0,0 +1,15 @@
+[cylc]
+    UTC mode = True
+    [[reference test]]
+        live mode suite timeout = PT1M
+[scheduling]
+    [[dependencies]]
+        graph = "t1 => t2"
+[runtime]
+    [[t1]]
+        script = """
+cylc broadcast -n 't2' -F "${CYLC_SUITE_DEF_PATH}/broadcast.rc" "${CYLC_SUITE_NAME}"
+sleep 1
+"""
+    [[t2]]
+        script = false

tests/broadcast/11-file-2.t

@@ -0,0 +1,28 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2017 NIWA
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test broadcast settings in 2 files
+. "$(dirname "$0")/test_header"
+set_test_number 2
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
+suite_run_ok "${TEST_NAME_BASE}" \
+    cylc run --debug --reference-test "${SUITE_NAME}"
+
+purge_suite "${SUITE_NAME}"
+exit

tests/broadcast/11-file-2/broadcast-1.rc

@@ -0,0 +1 @@
+script=printenv CYLC_FOOBAR

tests/broadcast/11-file-2/broadcast-2.rc

@@ -0,0 +1,6 @@
+[environment]
+    CYLC_FOOBAR="""
+foo
+bar
+baz
+"""

tests/broadcast/11-file-2/reference.log

@@ -0,0 +1,4 @@
+2017-01-12T14:46:57Z INFO - Initial point: 1
+2017-01-12T14:46:57Z INFO - Final point: 1
+2017-01-12T14:46:57Z INFO - [t1.1] -triggered off []
+2017-01-12T14:47:00Z INFO - [t2.1] -triggered off ['t1.1']

tests/broadcast/11-file-2/suite.rc

@@ -0,0 +1,18 @@
+[cylc]
+    UTC mode = True
+    [[reference test]]
+        live mode suite timeout = PT1M
+[scheduling]
+    [[dependencies]]
+        graph = "t1 => t2"
+[runtime]
+    [[t1]]
+        script = """
+cylc broadcast -n 't2' \
+    -F "${CYLC_SUITE_DEF_PATH}/broadcast-1.rc" \
+    -F "${CYLC_SUITE_DEF_PATH}/broadcast-2.rc" \
+    "${CYLC_SUITE_NAME}"
+sleep 1
+"""
+    [[t2]]
+        script = false