Merge pull request #2717 from matthewrmshin/7.7.x

7.7.2 take 1

bin/cylc

@@ -24,12 +24,14 @@ HELP_OPTS=('help' '--help' '-h' 'h' '?')
 get_version() {
     # If there is a version file present, use that
     if [[ -f "${CYLC_DIR}/VERSION" ]]; then
-        CYLC_VERSION="$(cat ${CYLC_DIR}/VERSION)"
-    # Otherwise, use git (if in a git repo) to determine version
-    elif [[ -d "${CYLC_DIR}/.git" ]]; then
-        CYLC_VERSION="$(cd ${CYLC_DIR} && git describe --abbrev=4 --always)"
+        CYLC_VERSION="$(cat "${CYLC_DIR}/VERSION")"
+    # Otherwise, use git (if in a git repo and not a bare clone)
+    # to determine version
+    elif git -C "${CYLC_DIR}" rev-parse && \
+            [[ -n "$(git -C "${CYLC_DIR}" rev-parse --show-toplevel)" ]]; then
+        CYLC_VERSION="$(cd "${CYLC_DIR}" && git describe --abbrev=4 --always)"
         # Append "-dirty" if there are uncommitted changes.
-        if [[ -n "$(cd ${CYLC_DIR} && git status \
+        if [[ -n "$(cd "${CYLC_DIR}" && git status \
                 --untracked-files=no --porcelain)" ]]; then
             CYLC_VERSION="${CYLC_VERSION}-dirty"
         fi

doc/src/cylc-user-guide/cug.tex

@@ -406,7 +406,7 @@ \subsection{Third-Party Software Packages}
 
 \begin{myitemize}
   \item {\bf PyGTK} - GUI toolkit \url{http://www.pygtk.org}.  {\em Note PyGTK
-    typically comes with your system Python. It is alledgedly quite
+    typically comes with your system Python. It is allegedly quite
     difficult to install if you need to do so for another Python version.}
     \item {\bf Graphviz} - graph layout engine (tested 2.36.0):
       \url{http://www.graphviz.org}.
@@ -510,7 +510,7 @@ \subsection{Software Bundled With Cylc}
 \subsection{Installing Cylc}
 \label{InstallCylc}
 
-Cylc releases can be downloaded from from \url{https://cylc.github.io/cylc}.
+Cylc releases can be downloaded from \url{https://cylc.github.io/cylc}.
 
 The wrapper script \lstinline=usr/bin/cylc= should be installed to
 the system executable search path (e.g.\ \lstinline=/usr/local/bin/=) and
@@ -656,7 +656,7 @@ \subsection{Parameterized Tasks as a Proxy for Cycling}
 loop over an integer parameter \lstinline@R = 0, 1, 2, ..., 11@ to define tasks
 \lstinline=model-R0, model-R1, model-R2, ...model-R11=, and the parameter
 values could be multiplied by the interval \lstinline=P1M= (one month) to get
-the start point point for the corresponding model run.
+the start point for the corresponding model run.
 
 This method is only good for smaller workflows of finite duration because every
 single task has to be mapped out in advance, and cylc has to be aware of all of
@@ -820,7 +820,7 @@ \subsection{Suite Passphrases}
 programs.
 
 Possession of a suite's passphrase file gives full control over it.
-Without it, the information avaiable to a client is determined by the suite's
+Without it, the information available to a client is determined by the suite's
 public access privilege level.
 
 For more on connection authentication, suite passphrases, and public access,
@@ -1189,7 +1189,7 @@ \subsection{Remote Tasks}
 running the suite server program, where a task job is submitted to run. It can
 be on the same machine running the suite or on another machine.
 
-A task remote account must satify several requirements:
+A task remote account must satisfy several requirements:
 \begin{myitemize}
 
 \item Non-interactive ssh must be enabled from the account running the suite
@@ -1495,7 +1495,7 @@ \subsection{Triggering Off Of Families}
 Any task triggering status qualified by \lstinline=-all= or
 \lstinline=-any=, for the members, can be used with a family trigger.
 For example, here's how to trigger \lstinline=bar= if all members
-of \lstinline=GREETERS= finish (succeed or fail) and any of them them
+of \lstinline=GREETERS= finish (succeed or fail) and any of them
 succeed:
 \lstset{language=suiterc}
 \begin{lstlisting}
@@ -2687,7 +2687,7 @@ \subsubsection{Graph Section Headings}
 For example, partial date-times can be excluded using the syntax:
 \lstset{language=suiterc}
 \begin{lstlisting}
-[[[ PT1H ! T12 ]]]          # Run hourly but not at 12:00 from the inital
+[[[ PT1H ! T12 ]]]          # Run hourly but not at 12:00 from the initial
                             # cycle point.
 [[[ T-00 ! (T00, T06, T12, T18) ]]]   # Run hourly but not at 00:00, 06:00,
                                       # 12:00, 18:00.
@@ -3759,7 +3759,7 @@ \subsubsection{Trigger Types}
 name of the target suite, the message that identifies this type of event to the
 suite, and an ID that distinguishes this particular event instance from others
 (the name of the target task or its current cycle point is not required). The
-event ID is just an arbitary string to cylc, but it typically identifies the
+event ID is just an arbitrary string to cylc, but it typically identifies the
 filename(s) of the latest dataset in some way. When the suite server program
 receives the external event notification it will trigger the next instance of
 any task waiting on that trigger (whatever its cycle point) and then broadcast
@@ -6102,7 +6102,7 @@ \subsection{Custom Job Submission Methods}
 \subsubsection{An Example}
 
 The following \lstinline=qsub.py= module overrides the built-in {\em pbs}
-batch system handler to to change the directive prefix from \lstinline=#PBS= to
+batch system handler to change the directive prefix from \lstinline=#PBS= to
 \lstinline=#QSUB=:
 
 \begin{lstlisting}

doc/src/suite-design-guide/general-principles.tex

@@ -386,7 +386,7 @@ \subsubsection{Inheritance}
 Sharing by inheritance of task families is recommended when more than a few
 configuration items are involved.
 
-The simplest application of inheritance is to set set global defaults in the
+The simplest application of inheritance is to set global defaults in the
 \lstinline=[[runtime]][root]= namespace that is inherited by all tasks.
 However, this should only be done for settings that really are used
 by the vast majority of tasks. Over-sharing of via root, particularly of

lib/cylc/batch_sys_manager.py

@@ -103,13 +103,15 @@
 """
 
 import os
+import json
 import shlex
 from shutil import rmtree
 from signal import SIGKILL
 import stat
 from subprocess import Popen, PIPE
 import sys
 import traceback
+
 from cylc.mkdir_p import mkdir_p
 from cylc.task_message import (
     CYLC_JOB_PID, CYLC_JOB_INIT_TIME, CYLC_JOB_EXIT_TIME, CYLC_JOB_EXIT,
@@ -119,23 +121,30 @@
     JOB_LOG_JOB, JOB_LOG_OUT, JOB_LOG_ERR, JOB_LOG_STATUS)
 from cylc.wallclock import get_current_time_string
 
+from parsec.OrderedDict import OrderedDict
 
-class JobPollContext(object):
-    """Context object for a job poll.
-
-    0 ctx.job_log_dir -- cycle/task/submit_num
-    1 ctx.batch_sys_name -- batch system name
-    2 ctx.batch_sys_job_id -- job ID in batch system
-    3 ctx.batch_sys_exit_polled -- 0 for false, 1 for true
-    4 ctx.run_status -- 0 for success, 1 for failure
-    5 ctx.run_signal -- signal received on run failure
-    6 ctx.time_submit_exit -- submit (exit) time
-    7 ctx.time_run -- run start time
-    8 ctx.time_run_exit -- run exit time
-
-    """
 
-    def __init__(self, job_log_dir):
+class JobPollContext(object):
+    """Context object for a job poll."""
+    CONTEXT_ATTRIBUTES = (
+        'job_log_dir',  # cycle/task/submit_num
+        'batch_sys_name',  # batch system name
+        'batch_sys_job_id',  # job id in batch system
+        'batch_sys_exit_polled',  # 0 for false, 1 for true
+        'run_status',  # 0 for success, 1 for failure
+        'run_signal',  # signal recieved on run failure
+        'time_submit_exit',  # submit (exit) time
+        'time_run',  # run start time
+        'time_run_exit',  # run exit time
+        'batch_sys_call_no_lines',  # number of lines in batch sys call stdout
+    )
+
+    __slots__ = CONTEXT_ATTRIBUTES + (
+        'pid',
+        'messages'
+    )
+
+    def __init__(self, job_log_dir, **attrs):
         self.job_log_dir = job_log_dir
         self.batch_sys_name = None
         self.batch_sys_job_id = None
@@ -146,26 +155,25 @@ def __init__(self, job_log_dir):
         self.time_submit_exit = None
         self.time_run = None
         self.time_run_exit = None
+        self.batch_sys_call_no_lines = None
         self.messages = []
 
+        if attrs:
+            for key, value in attrs.items():
+                if key in self.CONTEXT_ATTRIBUTES:
+                    setattr(self, key, value)
+                else:
+                    raise ValueError('Invalid kwarg "%s"' % key)
+
     def get_summary_str(self):
         """Return the poll context as a summary string delimited by "|"."""
-        items = []
-        for item in [
-                self.job_log_dir,
-                self.batch_sys_name,
-                self.batch_sys_job_id,
-                self.batch_sys_exit_polled,
-                self.run_status,
-                self.run_signal,
-                self.time_submit_exit,
-                self.time_run,
-                self.time_run_exit]:
-            if item is None:
-                items.append("")
-            else:
-                items.append(str(item))
-        return "|".join(items)
+        ret = OrderedDict()
+        for key in self.CONTEXT_ATTRIBUTES:
+            value = getattr(self, key)
+            if key == 'job_log_dir' or value is None:
+                continue
+            ret[key] = value
+        return '%s|%s' % (self.job_log_dir, json.dumps(ret))
 
 
 class BatchSysManager(object):
@@ -522,6 +530,7 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list):
             exp_pids = [ctx.pid for ctx in my_ctx_list if ctx.pid is not None]
             bad_pids.extend(exp_pids)
             items.append([self._get_sys("background"), exp_pids, bad_pids])
+        debug_messages = []
         for batch_sys, exp_ids, bad_ids in items:
             if hasattr(batch_sys, "get_poll_many_cmd"):
                 # Some poll commands may not be as simple
@@ -541,6 +550,8 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list):
                 return
             ret_code = proc.wait()
             out, err = proc.communicate()
+            debug_messages.append('%s - %s' % (
+                batch_sys, len(out.split('\n'))))
             sys.stderr.write(err)
             if (ret_code and hasattr(batch_sys, "POLL_CANT_CONNECT_ERR") and
                     batch_sys.POLL_CANT_CONNECT_ERR in err):
@@ -569,14 +580,17 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list):
                         except ValueError:
                             pass
 
+        debug_flag = False
         for ctx in my_ctx_list:
             ctx.batch_sys_exit_polled = int(
                 ctx.batch_sys_job_id in bad_job_ids)
             # Exited batch system, but process still running
             # This can happen to jobs in some "at" implementation
-            if (ctx.batch_sys_exit_polled and
-                    ctx.pid in exp_pids and ctx.pid not in bad_pids):
-                ctx.batch_sys_exit_polled = 0
+            if ctx.batch_sys_exit_polled and ctx.pid in exp_pids:
+                if ctx.pid not in bad_pids:
+                    ctx.batch_sys_exit_polled = 0
+                else:
+                    debug_flag = True
             # Add information to "job.status"
             if ctx.batch_sys_exit_polled:
                 try:
@@ -589,6 +603,9 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list):
                 except IOError as exc:
                     sys.stderr.write(str(exc) + "\n")
 
+        if debug_flag:
+            ctx.batch_sys_call_no_lines = ', '.join(debug_messages)
+
     def _job_submit_impl(
             self, job_file_path, batch_sys_name, submit_opts):
         """Helper for self.jobs_submit() and self.job_submit()."""

lib/cylc/cycling/__init__.py

@@ -433,12 +433,11 @@ def build_exclusions(self, excl_points):
         pass
 
     def __contains__(self, point):
-        """Checks to see if the Exclusions object contains a point
-        in any of the exclusion sequences.
+        """Return True if the provided point is in this exclusion.
 
         Args:
-            point (str): The time point to check lies in the
-                ISO8601Sequence object.
+            point (PointBase): The cycle point to check.
+
         """
         if point in self.exclusion_points:
             return True

lib/cylc/cycling/iso8601.py

@@ -295,11 +295,8 @@ class ISO8601Exclusions(ExclusionBase):
     may be used on this object to determine if a point is in the collection
     of exclusion sequences."""
 
-    __slots__ = ExclusionBase.__slots__ + ('p_iso_exclusions',)
-
     def __init__(self, excl_points, start_point, end_point=None):
         super(ISO8601Exclusions, self).__init__(start_point, end_point)
-        self.p_iso_exclusions = []
         self.build_exclusions(excl_points)
 
     def build_exclusions(self, excl_points):
@@ -310,7 +307,6 @@ def build_exclusions(self, excl_points):
                     str(point)) if point else None
                 if exclusion_point not in self.exclusion_points:
                     self.exclusion_points.append(exclusion_point)
-                    self.p_iso_exclusions.append(str(exclusion_point))
             except (AttributeError, TypeError, ValueError):
                 # Try making an ISO8601Sequence
                 exclusion = ISO8601Sequence(point, self.exclusion_start_point,
@@ -488,29 +484,29 @@ def get_nearest_prev_point(self, point):
         if self.is_on_sequence(point):
             return self.get_prev_point(point)
         p_iso_point = point_parse(point.value)
-        prev_iso_point = None
+        prev_cycle_point = None
 
         for recurrence_iso_point in self.recurrence:
+
             # Is recurrence point greater than aribitrary point?
-            if (
-                    recurrence_iso_point > p_iso_point or
-                    (self.exclusions and
-                     recurrence_iso_point in self.exclusions.p_iso_exclusions)
-            ):
+            if recurrence_iso_point > p_iso_point:
                 break
-            prev_iso_point = recurrence_iso_point
-        if prev_iso_point is None:
+            recurrence_cycle_point = ISO8601Point(str(recurrence_iso_point))
+            if self.exclusions and recurrence_cycle_point in self.exclusions:
+                break
+            prev_cycle_point = recurrence_cycle_point
+
+        if prev_cycle_point is None:
             return None
-        nearest_point = ISO8601Point(str(prev_iso_point))
-        if nearest_point == point:
+        if prev_cycle_point == point:
             raise SequenceDegenerateError(
                 self.recurrence, SuiteSpecifics.DUMP_FORMAT,
-                nearest_point, point
+                prev_cycle_point, point
             )
         # Check all exclusions
-        if self.exclusions and nearest_point in self.exclusions:
-            return self.get_prev_point(nearest_point)
-        return nearest_point
+        if self.exclusions and prev_cycle_point in self.exclusions:
+            return self.get_prev_point(prev_cycle_point)
+        return prev_cycle_point
 
     def get_next_point(self, point):
         """Return the next point > p, or None if out of bounds."""

lib/cylc/gui/view_graph.py

@@ -108,7 +108,11 @@ def on_url_clicked(self, _, task_id, event):
             # base graph node
             task_id = task_id[len(self.t.PREFIX_BASE):]
 
-        name, point_string = TaskID.split(task_id)
+        try:
+            name, point_string = TaskID.split(task_id)
+        except ValueError:
+            # not a task -> skip
+            return False
 
         menu = gtk.Menu()
         menu_root = gtk.MenuItem(task_id)

lib/cylc/network/httpclient.py

@@ -385,18 +385,22 @@ def call_server_impl(self, url, method, payload):
     def _call_server_impl_requests(self, url, method, payload):
         """Call server with "requests" library."""
         import requests
+        # Filter InsecureRequestWarning from urlib3. We use verify=False
+        # deliberately (and safely) for anonymous access.
+        from requests.packages.urllib3.exceptions import InsecureRequestWarning
+        warnings.simplefilter("ignore", InsecureRequestWarning)
         # Filter security warnings from urllib3 on python <2.7.9. Obviously, we
         # want to upgrade, but some sites have to run cylc on platforms with
         # python <2.7.9. On those platforms, these warnings serve no purpose
         # except to annoy or confuse users.
         if sys.version_info < (2, 7, 9):
             try:
                 from requests.packages.urllib3.exceptions import (
-                    SecurityWarning)
+                    InsecurePlatformWarning)
             except ImportError:
                 pass
             else:
-                warnings.simplefilter("ignore", SecurityWarning)
+                warnings.simplefilter("ignore", InsecurePlatformWarning)
             try:
                 from requests.packages.urllib3.exceptions import (
                     SNIMissingWarning)

lib/cylc/network/httpserver.py

@@ -51,8 +51,6 @@ class HTTPServer(object):
 
     API = 2
     LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s"
-    RE_MESSAGE_TIME = re.compile(
-        r'\A(.+) at (' + RE_DATE_TIME_FORMAT_EXTENDED + r')\Z', re.DOTALL)
 
     def __init__(self, suite):
         # Suite only needed for back-compat with old clients (see below):
@@ -204,6 +202,8 @@ class SuiteRuntimeService(object):
     LOG_IDENTIFY_TMPL = '[client-identify] %d id requests in PT%dS'
     LOG_FORGET_TMPL = '[client-forget] %s'
     LOG_CONNECT_ALLOWED_TMPL = "[client-connect] %s@%s:%s privilege='%s' %s"
+    RE_MESSAGE_TIME = re.compile(
+        r'\A(.+) at (' + RE_DATE_TIME_FORMAT_EXTENDED + r')\Z', re.DOTALL)
 
     def __init__(self, schd):
         self.schd = schd