From e72b70455c28e8d0d92f731570021a67c1d11de9 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 5 Sep 2019 09:31:02 +1200 Subject: [PATCH 1/7] Add back-port of master graph command. Handles `--reference` text-format only, without pygtk. --- bin/cylc | 4 + bin/cylc-help | 2 + bin/cylc-ref-graph | 240 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100755 bin/cylc-ref-graph diff --git a/bin/cylc b/bin/cylc index d756fc18d4f..42344fdb60e 100755 --- a/bin/cylc +++ b/bin/cylc @@ -75,6 +75,10 @@ help_util() { local COMMAND="${CYLC_HOME_BIN}/cylc-graph" exec "${COMMAND}" "--help" fi + if [[ "$@" == "ref-graph" ]]; then + local COMMAND="${CYLC_HOME_BIN}/cylc-ref-graph" + exec "${COMMAND}" "--help" + fi # For help command/option with no args if [[ $# == 0 && "${HELP_OPTS[*]} " == *"$UTIL"* ]]; then exec "${CYLC_HOME_BIN}/cylc-help" diff --git a/bin/cylc-help b/bin/cylc-help index 04eac4e1b35..3415d17f23a 100755 --- a/bin/cylc-help +++ b/bin/cylc-help @@ -311,6 +311,7 @@ preparation_commands['5to6'] = ['5to6'] preparation_commands['list'] = ['list', 'ls'] preparation_commands['search'] = ['search', 'grep'] preparation_commands['graph'] = ['graph'] +preparation_commands['ref-graph'] = ['ref-graph'] preparation_commands['graph-diff'] = ['graph-diff'] preparation_commands['diff'] = ['diff', 'compare'] preparation_commands['jobscript'] = ['jobscript'] @@ -380,6 +381,7 @@ comsum['validate'] = 'Parse and validate suite definitions' comsum['5to6'] = 'Improve the cylc 6 compatibility of a cylc 5 suite file' comsum['search'] = 'Search in suite definitions' comsum['graph'] = 'Plot suite dependency graphs and runtime hierarchies' +comsum['ref-graph'] = 'Print text-format "reference graphs" without GTK' comsum['graph-diff'] = 'Compare two suite dependencies or runtime hierarchies' comsum['diff'] = 'Compare two suite definitions and print differences' # information diff --git a/bin/cylc-ref-graph b/bin/cylc-ref-graph new file mode 100755 index 00000000000..29e02bc03e1 --- /dev/null +++ b/bin/cylc-ref-graph @@ -0,0 +1,240 @@ +#!/usr/bin/env python2 +# +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Usage: + cylc graph SUITE [START] [STOP] + +Implement the old ``cylc graph --reference command`` for producing a +text-format representation of a suite graph. + +THIS IS A BACK-PORT OF the Python 3 bin/cylc-graph FOR CYLC 8, TO GENERATE +TEXT-FORMAT "REFERENCE GRAPHS" WITHOUT THE NEED FOR PYGTK TO BE INSTALLED.""" + +from cylc.config import SuiteConfig +from cylc.cycling.loader import get_point +from cylc.option_parsers import CylcOptionParser as COP +from cylc.suite_srv_files_mgr import ( + SuiteSrvFilesManager, SuiteServiceFileError) +from cylc.templatevars import load_template_vars + + +def sort_integer_node(item): + """Return sort tokens for nodes with cyclepoints in integer format. + + Example: + >>> sort_integer_node('foo.11') + ('foo', 11) + + """ + name, point = item.split('.') + return (name, int(point)) + + +def sort_integer_edge(item): + """Return sort tokens for edges with cyclepoints in integer format. + + Example: + >>> sort_integer_edge(('foo.11', 'foo.12', None)) + (('foo', 11), ('foo', 12)) + >>> sort_integer_edge(('foo.11', None , None)) + (('foo', 11), ('', 0)) + + """ + + return ( + sort_integer_node(item[0]), + sort_integer_node(item[1]) if item[1] else ('', 0) + ) + + +def sort_datetime_edge(item): + """Return sort tokens for edges with cyclepoints in ISO8601 format. + + Example: + >>> sort_datetime_edge(('a', None, None)) + ('a', '') + + """ + return (item[0], item[1] or '') + + +def get_cycling_bounds(config, start_point=None, stop_point=None): + """Determine the start and stop points for graphing a suite.""" + # default start and stop points to values in the visualization section + if not start_point: + start_point = config.cfg['visualization']['initial cycle point'] + if not stop_point: + viz_stop_point = config.cfg['visualization']['final cycle point'] + if viz_stop_point: + stop_point = viz_stop_point + + # don't allow stop_point before start_point + if stop_point is not None: + if get_point(stop_point) < get_point(start_point): + # NOTE: we need to cast with get_point for this comparison due to + # ISO8061 extended datetime formats + stop_point = start_point + else: + stop_point = stop_point + else: + stop_point = None + + return start_point, stop_point + + +def graph_workflow(config, start_point=None, stop_point=None, ungrouped=False, + show_suicide=False): + """Implement ``cylc-graph --reference``.""" + # set sort keys based on cycling mode + if config.cfg['scheduling']['cycling mode'] == 'integer': + # integer sorting + node_sort = sort_integer_node + edge_sort = sort_integer_edge + else: + # datetime sorting + node_sort = None # lexicographically sortable + edge_sort = sort_datetime_edge + + # get graph + start_point, stop_point = get_cycling_bounds( + config, start_point, stop_point) + graph = config.get_graph_raw( + start_point, stop_point, ungroup_all=ungrouped) + if not graph: + return + + edges = ( + (left, right) + for left, right, _, suicide, _ in graph + if right + if show_suicide or not suicide + ) + for left, right in sorted(set(edges), key=edge_sort): + print('edge "%s" "%s"' % (left, right)) + + print('graph') + + # print nodes + nodes = ( + node + for left, right, _, suicide, _ in graph + for node in (left, right) + if node + if show_suicide or not suicide + ) + for node in sorted(set(nodes), key=node_sort): + print('node "%s" "%s"' % (node, node.replace('.', r'\n'))) + + print('stop') + + +def graph_inheritance(config): + """Implement ``cylc-graph --reference --namespaces``.""" + edges = set() + nodes = set() + for namespace, tasks in config.get_parent_lists().items(): + for task in tasks: + edges.add((task, namespace)) + nodes.add(task) + + for namespace in config.get_parent_lists(): + nodes.add(namespace) + + for edge in sorted(edges): + print('edge %s %s' % edge) + + print('graph') + + for node in sorted(nodes): + print('node %s %s' % (node, node)) + + print('stop') + + +def get_config(suite, opts, template_vars=None): + """Return a SuiteConfig object for the provided reg / path.""" + try: + suiterc = SuiteSrvFilesManager().get_suite_rc(suite) + except SuiteServiceFileError: + # could not find suite, assume we have been given a path instead + suiterc = suite + suite = 'test' + return SuiteConfig(suite, suiterc, opts, template_vars=template_vars) + + +def get_option_parser(): + """CLI.""" + parser = COP( + __doc__, jset=True, prep=True, + argdoc=[ + ('[SUITE]', 'Suite name or path'), + ('[START]', 'Initial cycle point ' + '(default: suite initial point)'), + ('[STOP]', 'Final cycle point ' + '(default: initial + 3 points)')]) + + parser.add_option( + '-u', '--ungrouped', + help='Start with task families ungrouped (the default is grouped).', + action='store_true', default=False, dest='ungrouped') + + parser.add_option( + '-n', '--namespaces', + help='Plot the suite namespace inheritance hierarchy ' + '(task run time properties).', + action='store_true', default=False, dest='namespaces') + + parser.add_option( + '-r', '--reference', + help='Output in a sorted plain text format for comparison purposes.', + action='store_true', default=True, dest='reference') + + parser.add_option( + '--show-suicide', + help='Show suicide triggers. They are not shown by default, unless ' + 'toggled on with the tool bar button.', + action='store_true', default=False, dest='show_suicide') + + parser.add_option( + '--icp', action='store', default=None, metavar='CYCLE_POINT', help=( + 'Set initial cycle point. Required if not defined in suite.rc.')) + + return parser + + +def main(opts, suite=None, start=None, stop=None): + """Implement ``cylc graph``.""" + if opts.ungrouped and opts.namespaces: + raise Exception('Cannot combine --ungrouped and --namespaces.') + if not opts.reference: + raise Exception('Only the --reference use cases are supported') + + template_vars = load_template_vars( + opts.templatevars, opts.templatevars_file) + + config = get_config(suite, opts, template_vars=template_vars) + if opts.namespaces: + graph_inheritance(config) + else: + graph_workflow(config, start, stop, ungrouped=opts.ungrouped, + show_suicide=opts.show_suicide) + + +if __name__ == '__main__': + options, args = get_option_parser().parse_args() + suite, _ = SuiteSrvFilesManager().parse_suite_arg(options, args[0]) + main(options, suite) From ea7a1f5d6754e10fe71995201b5f6644514a08c7 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 5 Sep 2019 10:24:06 +1200 Subject: [PATCH 2/7] Documented confusing PyGTK installation. --- doc/src/installation.rst | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/src/installation.rst b/doc/src/installation.rst index 3474aa311a9..8c1438cdf21 100644 --- a/doc/src/installation.rst +++ b/doc/src/installation.rst @@ -25,12 +25,21 @@ The following packages are highly recommended, but are technically optional as you can construct and run suites without dependency graph visualisation or the Cylc GUIs: -- `PyGTK `_ - GUI toolkit. +- `PyGTK `_ - Python bindings for the GTK+ GUI toolkit. .. note:: - PyGTK typically comes with your system Python. It is allegedly quite - difficult to install if you need to do so for another Python version. + PyGTK typically comes with your system Python 2 version. It is allegedly + quite difficult to install if you need to do so for another Python + version. At time of writing, for instance, there are no functional PyGTK + conda packages available. + + Note that **we need to do ``import gtk`` in Python, not ``import pygtk``**. + + In Centos 7.6, for example, the Cylc GUIs run "out of the box" with the + system-installed Python 2.7.5. Under the hood, the Python “gtk” package is + provided by the “pygtk2” yum package. (The “pygtk” Python module, which we + don't need, is supplied by the “pygobject2” yum package). - `Graphviz `_ - graph layout engine (tested 2.36.0) - `Pygraphviz `_ - Python Graphviz interface @@ -39,6 +48,12 @@ the Cylc GUIs: - python-devel - graphviz-devel + .. note:: + + The ``cylc graph`` command for static workflow visualization requires + PyGTK, but we provide a separate ``cylc ref-graph`` command to print + out a simple text-format "reference graph" without PyGTK. + The Cylc Review service does not need any additional packages. The following packages are necessary for running all the tests in Cylc: From 7cf7b437fe61bd68cf4a76e1f2a3b583d299ca53 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 5 Sep 2019 10:44:23 +1200 Subject: [PATCH 3/7] Update change log. --- CHANGES.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6fb80db0208..6949e37e9bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,17 @@ pages**. - cylc-8 (master branch, Python 3 - not yet released) does not bundle Jinja2, and uses the fixed version 2.10.1. +------------------------------------------------------------------------------- +## __cylc-7.8.5 (2019-Q4?)__ + + +### Enhancements + +[#3349](https://github.com/cylc/cylc-flow/pull/3349) - new command `cylc +ref-graph` to generate text-format "reference graphs" without PyGTK (back-port +from Python 3 master for Cylc 8). + + ------------------------------------------------------------------------------- ## __cylc-7.8.4 (2019-09-04)__ From 748f2ac19b2c68927f3f395490f77780d720eac8 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 5 Sep 2019 21:23:01 +1200 Subject: [PATCH 4/7] Update bin/cylc-ref-graph Co-Authored-By: Sadie L. Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> --- bin/cylc-ref-graph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cylc-ref-graph b/bin/cylc-ref-graph index 29e02bc03e1..66784a6b0e0 100755 --- a/bin/cylc-ref-graph +++ b/bin/cylc-ref-graph @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Usage: - cylc graph SUITE [START] [STOP] + cylc ref-graph SUITE [START] [STOP] Implement the old ``cylc graph --reference command`` for producing a text-format representation of a suite graph. From 0ccd3861c23763e010e803033fe30968e64a838a Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 6 Sep 2019 08:30:18 +1200 Subject: [PATCH 5/7] Address review feedback. --- bin/cylc-ref-graph | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/cylc-ref-graph b/bin/cylc-ref-graph index 66784a6b0e0..8dc72535786 100755 --- a/bin/cylc-ref-graph +++ b/bin/cylc-ref-graph @@ -21,6 +21,8 @@ Implement the old ``cylc graph --reference command`` for producing a text-format representation of a suite graph. +The `--reference` flag is optional here. + THIS IS A BACK-PORT OF the Python 3 bin/cylc-graph FOR CYLC 8, TO GENERATE TEXT-FORMAT "REFERENCE GRAPHS" WITHOUT THE NEED FOR PYGTK TO BE INSTALLED.""" @@ -88,10 +90,6 @@ def get_cycling_bounds(config, start_point=None, stop_point=None): # NOTE: we need to cast with get_point for this comparison due to # ISO8061 extended datetime formats stop_point = start_point - else: - stop_point = stop_point - else: - stop_point = None return start_point, stop_point From 1820b1d9d450b9c142369f18e0bdd58e115a223f Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 6 Sep 2019 08:30:34 +1200 Subject: [PATCH 6/7] Add a note to cylc-graph command help. --- bin/cylc-graph | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/cylc-graph b/bin/cylc-graph index 4ed0d9ae415..0550c5ed4ce 100755 --- a/bin/cylc-graph +++ b/bin/cylc-graph @@ -26,7 +26,11 @@ Plot SUITE dependencies to a file FILE with a extension-derived format. If FILE endswith ".png", output in PNG format, etc. -Plot suite dependency graphs in an interactive graph viewer. +Plot suite dependency graphs in an interactive graph viewer, or (with +"--output-file") directly to image file. + +See also "cylc ref-graph" to generate the plain text "reference" graph format +without the need for PyGTK to be installed. If START is given it overrides "[visualization] initial cycle point" to determine the start point of the graph, which defaults to the suite initial From c61579a7fcb54ada77b58f9ca7d8ff068aec8907 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 6 Sep 2019 08:52:43 +1200 Subject: [PATCH 7/7] Add test for cylc-ref-graph. --- tests/cylc-graph-diff/00-simple.t | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/cylc-graph-diff/00-simple.t b/tests/cylc-graph-diff/00-simple.t index 1ca3401aded..fa6f9bef46f 100644 --- a/tests/cylc-graph-diff/00-simple.t +++ b/tests/cylc-graph-diff/00-simple.t @@ -18,7 +18,7 @@ # Test cylc graph-diff for two suites. . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 24 +set_test_number 27 #------------------------------------------------------------------------------- install_suite $TEST_NAME_BASE-control $TEST_NAME_BASE-control CONTROL_SUITE_NAME=$SUITE_NAME @@ -160,6 +160,18 @@ node "foo.20140810T0000Z" "foo\n20140810T0000Z" stop __OUT__ cmp_ok "$TEST_NAME.stderr"