From d29dc0a6c67bc9a888787985c9715a75c1da72a7 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Tue, 22 Jul 2014 16:32:10 +0100 Subject: [PATCH 01/14] draft of per-task-instance cleanup-cutoff --- lib/cylc/config.py | 65 ++++++++++++-------- lib/cylc/cycling/integer.py | 10 ++++ lib/cylc/cycling/iso8601.py | 41 +++++++++++-- lib/cylc/cycling/loader.py | 11 ++++ lib/cylc/graphnode.py | 44 ++++++++++---- lib/cylc/taskdef.py | 82 +++++++++++++++++++------- lib/cylc/trigger.py | 21 ++++--- tests/cyclers/monthly_complex/suite.rc | 21 +++++++ 8 files changed, 222 insertions(+), 73 deletions(-) create mode 100644 tests/cyclers/monthly_complex/suite.rc diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 470ced177ef..86833e75ea2 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -19,10 +19,11 @@ import re, os, sys import taskdef from cylc.cfgspec.suite import get_suitecfg -from cylc.cycling.loader import (get_point, get_interval_cls, - get_sequence, get_sequence_cls, - init_cyclers, INTEGER_CYCLING_TYPE, - get_backwards_compatibility_mode) +from cylc.cycling.loader import ( + get_point, get_point_relative, get_interval_cls, get_sequence, + get_sequence_cls, init_cyclers, INTEGER_CYCLING_TYPE, + get_backwards_compatibility_mode +) from envvar import check_varnames, expandvars from copy import deepcopy, copy from output import outputx @@ -771,8 +772,9 @@ def process_directories(self): os.environ['CYLC_SUITE_REG_PATH'] = RegPath( self.suite ).get_fpath() os.environ['CYLC_SUITE_DEF_PATH'] = self.fdir - def set_trigger( self, task_name, right, output_name=None, offset=None, - cycle_point=None, suicide=False, base_interval=None ): + def set_trigger( self, task_name, right, output_name=None, + offset_string=None, cycle_point=None, + suicide=False, base_interval=None ): trig = triggerx(task_name) trig.set_suicide(suicide) if output_name: @@ -809,8 +811,9 @@ def set_trigger( self, task_name, right, output_name=None, offset=None, # default: task succeeded trig.set_type( 'succeeded' ) - if offset: - trig.set_offset( str(offset) ) # TODO ISO - CONSISTENT SET_OFFSET INPUT + if offset_string: + # TODO ISO - CONSISTENT SET_OFFSET INPUT + trig.set_offset_string( offset_string ) if cycle_point: trig.set_cycle_point( cycle_point ) @@ -1240,7 +1243,7 @@ def generate_taskdefs( self, line, left_nodes, right, ttype, section, seq, raise SuiteConfigError, str(x) name = my_taskdef_node.name - offset = my_taskdef_node.offset + offset_string = my_taskdef_node.offset_string if name not in self.cfg['runtime']: # naked dummy task, implicit inheritance from root @@ -1280,19 +1283,20 @@ def generate_taskdefs( self, line, left_nodes, right, ttype, section, seq, 'status' : self.suite_polling_tasks[name][2] } if not my_taskdef_node.is_absolute: - if offset: + if offset_string: if flags.back_comp_cycling: # Implicit cycling means foo[T+6] generates a +6 sequence. - if str(offset) in offset_seq_map: - seq_offset = offset_seq_map[str(offset)] + if offset_string in offset_seq_map: + seq_offset = offset_seq_map[offset_string] else: seq_offset = get_sequence( section, self.cfg['scheduling']['initial cycle point'], self.cfg['scheduling']['final cycle point'] ) - seq_offset.set_offset(offset) - offset_seq_map[str(offset)] = seq_offset + seq_offset.set_offset( + -get_interval(offset_string)) + offset_seq_map[offset_string] = seq_offset self.taskdefs[name].add_sequence( seq_offset, is_implicit=True) # We don't handle implicit cycling in new-style cycling. @@ -1326,22 +1330,31 @@ def generate_triggers( self, lexpression, left_nodes, right, seq, suicide ): # (GraphNodeError checked above) cycle_point = None lnode = graphnode(left, base_interval=base_interval) - if lnode.intercycle: - self.taskdefs[lnode.name].intercycle = True - if (self.taskdefs[lnode.name].intercycle_offset is None or ( - lnode.offset is not None and - lnode.offset > - self.taskdefs[lnode.name].intercycle_offset)): - self.taskdefs[lnode.name].intercycle_offset = lnode.offset + l_taskdef = self.taskdefs[lnode.name] if lnode.offset_is_from_ict: + print "Get point relative", lnode.offset_string, l_taskdef.ict + first_point = get_point_relative( + lnode.offset_string, l_taskdef.ict) last_point = seq.get_stop_point() - first_point = self.taskdefs[lnode.name].ict - lnode.offset - if first_point and last_point is not None: - self.taskdefs[lnode.name].intercycle_offset = (last_point - first_point) + if last_point is None: + # This dependency persists for the whole suite run. + l_taskdef.intercycle_offsets.append( + (None, seq)) else: - self.taskdefs[lnode.name].intercycle_offset = None + l_taskdef.intercycle_offsets.append( + (str(-(last_point - first_point)), seq)) cycle_point = first_point - trigger = self.set_trigger( lnode.name, right, lnode.output, lnode.offset, cycle_point, suicide, seq.get_interval() ) + elif lnode.intercycle: + l_taskdef.intercycle = True + if lnode.offset_is_irregular: + offset_tuple = (lnode.offset_string, seq) + else: + offset_tuple = (lnode.offset_string, None) + l_taskdef.intercycle_offsets.append(offset_tuple) + trigger = self.set_trigger( + lnode.name, right, lnode.output, lnode.offset_string, + cycle_point, suicide, seq.get_interval() + ) if not trigger: continue if not conditional: diff --git a/lib/cylc/cycling/integer.py b/lib/cylc/cycling/integer.py index 2b4ff9c6027..0765fbb9922 100755 --- a/lib/cylc/cycling/integer.py +++ b/lib/cylc/cycling/integer.py @@ -377,6 +377,10 @@ def get_first_point( self, point ): point = self.get_next_point( point ) return point + def get_start_point( self ): + """Return the first point in this sequence, or None.""" + return self.p_start + def get_stop_point( self ): """Return the last point in this sequence, or None if unbounded.""" return self.p_stop @@ -396,6 +400,12 @@ def init_from_cfg(cfg): pass +def get_point_relative(offset_string, base_point): + """Create a point from offset_string applied to base_point.""" + # TODO ISO: needs to be done somehow? Is it meaningful? + raise NotImplementedError() + + if __name__ == '__main__': r = IntegerSequence( 'R/1/P3', 1, 10 ) diff --git a/lib/cylc/cycling/iso8601.py b/lib/cylc/cycling/iso8601.py index 97b1287735e..a98bce98202 100755 --- a/lib/cylc/cycling/iso8601.py +++ b/lib/cylc/cycling/iso8601.py @@ -51,6 +51,7 @@ # The following must be set by calling the init_from_cfg function. # TODO: this is yukky. Is there a better alternative? +abbrev_util = None point_parser = None NUM_EXPANDED_YEAR_DIGITS = None DUMP_FORMAT = None @@ -288,15 +289,14 @@ def __init__(self, dep_section, context_start_point=None, self.custom_point_parse_function = None if DUMP_FORMAT == PREV_DATE_TIME_FORMAT: self.custom_point_parse_function = point_parse - - self.time_parser = CylcTimeParser( + self.abbrev_util = CylcTimeParser( self.context_start_point, self.context_end_point, num_expanded_year_digits=NUM_EXPANDED_YEAR_DIGITS, dump_format=DUMP_FORMAT, custom_point_parse_function=self.custom_point_parse_function, assumed_time_zone=ASSUMED_TIME_ZONE ) - self.recurrence = self.time_parser.parse_recurrence(i) + self.recurrence = self.abbrev_util.parse_recurrence(i) self.step = ISO8601Interval(str(self.recurrence.interval)) self.value = str(self.recurrence) @@ -384,7 +384,7 @@ def get_next_point_on_sequence(self, point): return result def get_first_point( self, point): - """Return the first point >= to poing, or None if out of bounds.""" + """Return the first point >= to point, or None if out of bounds.""" try: return ISO8601Point(self._cached_first_point_values[point.value]) except KeyError: @@ -401,6 +401,12 @@ def get_first_point( self, point): return ISO8601Point(first_point_value) return None + def get_start_point( self ): + """Return the first point in this sequence, or None.""" + for recurrence_iso_point in self.recurrence: + return ISO8601Point(str(recurrence_iso_point)) + return None + def get_stop_point( self ): """Return the last point in this sequence, or None if unbounded.""" if (self.recurrence.repetitions is not None or ( @@ -505,6 +511,7 @@ def init_from_cfg(cfg): def init(num_expanded_year_digits=0, custom_dump_format=None, time_zone=None, assume_utc=False, cycling_mode=None): """Initialise global variables (yuk).""" + global abbrev_util global point_parser global DUMP_FORMAT global NUM_EXPANDED_YEAR_DIGITS @@ -545,6 +552,32 @@ def init(num_expanded_year_digits=0, custom_dump_format=None, time_zone=None, dump_format=DUMP_FORMAT, assumed_time_zone=time_zone_hours_minutes ) + custom_point_parse_function = None + if DUMP_FORMAT == PREV_DATE_TIME_FORMAT: + custom_point_parse_function = point_parse + abbrev_util = CylcTimeParser( + None, None, + num_expanded_year_digits=NUM_EXPANDED_YEAR_DIGITS, + dump_format=DUMP_FORMAT, + custom_point_parse_function=custom_point_parse_function, + assumed_time_zone=ASSUMED_TIME_ZONE + ) + + +def get_point_relative(offset_string, base_point): + """Create a point from offset_string applied to base_point.""" + try: + interval = ISO8601Interval( + str(interval_parse(offset_string))) + except Exception: + pass + else: + return base_point + interval + return ISO8601Point(str( + abbrev_util.parse_timepoint( + offset_string, context_point=_point_parse(base_point.value) + ) + )) def interval_parse(interval_string): diff --git a/lib/cylc/cycling/loader.py b/lib/cylc/cycling/loader.py index 07142ddaebc..83ab7ab1a6b 100755 --- a/lib/cylc/cycling/loader.py +++ b/lib/cylc/cycling/loader.py @@ -33,6 +33,11 @@ POINTS = {INTEGER_CYCLING_TYPE: integer.IntegerPoint, ISO8601_CYCLING_TYPE: iso8601.ISO8601Point} +POINT_RELATIVE_GETTERS = { + INTEGER_CYCLING_TYPE: integer.get_point_relative, + ISO8601_CYCLING_TYPE: iso8601.get_point_relative +} + INTERVALS = {INTEGER_CYCLING_TYPE: integer.IntegerInterval, ISO8601_CYCLING_TYPE: iso8601.ISO8601Interval} @@ -73,6 +78,12 @@ def get_point_cls(cycling_type=None): return POINTS[cycling_type] +def get_point_relative(*args, **kwargs): + """Return a point from an offset expression and a base point.""" + cycling_type = kwargs.pop("cycling_type", DefaultCycler.TYPE) + return POINT_RELATIVE_GETTERS[cycling_type](*args, **kwargs) + + def get_interval(*args, **kwargs): if args[0] is None: return None diff --git a/lib/cylc/graphnode.py b/lib/cylc/graphnode.py index 72629e49cd1..6848eb5e827 100755 --- a/lib/cylc/graphnode.py +++ b/lib/cylc/graphnode.py @@ -58,6 +58,19 @@ (:[\w-]+|)$ # Optional type (e.g. :succeed) """, re.X) +# A potentially non-regular offset, such as foo[01T+P1W]. +IRREGULAR_OFFSET_RE = re.compile( + r"""^ # Start of string + ( # Begin group + ..+ # EITHER: Two or more characters + [+-]P # Then either +P or -P for start of duration + .* # Then anything for the rest of the duration + | # OR: + [^P]+ # No 'P' characters anywhere (e.g. T00). + ) # End group + $ # End of string + """, re.X) + class GraphNodeError( Exception ): """ Attributes: @@ -83,6 +96,7 @@ def __init__( self, node, base_interval=None ): # or relative to initial cycle point: foo[^+P1D] self.offset_is_from_ict = False + self.offset_is_irregular = False self.is_absolute = False is_prev_cycling_format = False @@ -91,7 +105,7 @@ def __init__( self, node, base_interval=None ): if m: # node looks like foo[^], foo[^-P4D], foo[^]:fail, etc. self.is_absolute = True - name, offset, outp = m.groups() + name, offset_string, outp = m.groups() self.offset_is_from_ict = True sign = "" prev_format = False @@ -99,7 +113,7 @@ def __init__( self, node, base_interval=None ): m = re.match( NODE_ISO_RE, node ) if m: # node looks like foo, foo:fail, foo[-PT6H], foo[-P4D]:fail... - name, offset, outp = m.groups() + name, offset_string, outp = m.groups() sign = "" prev_format = False else: @@ -108,7 +122,7 @@ def __init__( self, node, base_interval=None ): raise GraphNodeError( 'Illegal graph node: ' + node ) is_prev_cycling_format = True # node looks like foo[T-6], foo[T-12]:fail... - name, sign, offset, outp = m.groups() + name, sign, offset_string, outp = m.groups() prev_format = True if outp: @@ -123,23 +137,27 @@ def __init__( self, node, base_interval=None ): else: raise GraphNodeError( 'Illegal graph node: ' + node ) - if self.offset_is_from_ict and not offset: - offset = str(get_interval_cls().get_null()) - if offset: + if self.offset_is_from_ict and not offset_string: + offset_string = str(get_interval_cls().get_null()) + if offset_string: self.intercycle = True if prev_format: - self.offset = base_interval.get_inferred_child(offset) - if sign == "+": - self.offset = (-self.offset).standardise() + self.offset_string = str( + base_interval.get_inferred_child(offset_string)) else: - self.offset = (-get_interval(offset)).standardise() + if IRREGULAR_OFFSET_RE.search(offset_string): + self.offset_string = offset_string + self.offset_is_irregular = True + else: + self.offset_string = str( + (get_interval(offset_string)).standardise()) else: self.intercycle = False - self.offset = None + self.offset_string = None if not flags.back_comp_cycling and is_prev_cycling_format: raise GraphNodeError( - 'Illegal graph offset (new-style cycling): ' + str(offset) + - ' should be ' + str(self.offset) + 'Illegal graph offset (new-style cycling): ' + + '%s should be %s' % (offset_string, self.offset_string) ) diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 87c5b868b84..deb29cc7c60 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -31,7 +31,8 @@ import TaskID from task_output_logs import logfiles from parsec.OrderedDict import OrderedDict -from cycling.loader import get_interval_cls +from cycling.loader import get_interval_cls, get_point_relative + class Error( Exception ): """base class for exceptions in this module.""" @@ -63,7 +64,7 @@ def __init__( self, name, rtcfg, run_mode, ict ): # some defaults self.intercycle = False - self.intercycle_offset = get_interval_cls().get_null() + self.intercycle_offsets = [] self.sequential = False self.cycling = False self.modifiers = [] @@ -99,6 +100,47 @@ def add_sequence( self, sequence, is_implicit=False ): if is_implicit: self.implicit_sequences.append( sequence ) + def extract_cleanup_cutoff( self, my_point, offset_sequence_tuples ): + """Extract a validity cutoff point from dependent task information.""" + print "Extract cleanup cutoff", self.name, my_point + if not offset_sequence_tuples: + print " not offset_seq_tuples: None" + return None + cutoff_points = [] + for offset_string, sequence in offset_sequence_tuples: + print " offset_string, sequence", offset_string, sequence.value + if offset_string is None: + # This indicates a dependency across the whole suite run. + return None + if sequence is None: + cutoff_points.append( + get_point_relative(offset_string, my_point)) + print " cutoff point candidate:", cutoff_points[-1] + continue + dependent_point = sequence.get_start_point() + + matching_dependent_points = [] + while dependent_point is not None: + target_point = ( + get_point_relative(offset_string, dependent_point)) + print " dependent, target:", dependent_point, target_point + if target_point > my_point: + # Assume monotonic (target_point can never jump back). + break + if target_point == my_point: + matching_dependent_points.append(dependent_point) + print " cutoff point candidate:", cutoff_points[-1] + dependent_point = sequence.get_next_point_on_sequence( + dependent_point) + if matching_dependent_points: + # Choose the largest of the dependent points. + cutoff_points.append(matching_dependent_points[-1]) + if cutoff_points: + print " cutoff point choice:", max(cutoff_points) + return max(cutoff_points) + print " no cutoff points:", None + return None + def time_trans( self, strng, hours=False ): # Time unit translation. # THIS IS NOT CURRENTLY USED, but may be useful in the future. @@ -221,16 +263,18 @@ def tclass_add_prerequisites( sself, tag ): if trig.cycling and not sequence.is_valid( sself.tag ): # This trigger is not used in current cycle continue - if self.ict is None or \ - trig.evaluation_offset is None or \ - ( tag - trig.evaluation_offset ) >= self.ict: - # i.c.t. can be None after a restart, if one - # is not specified in the suite definition. - - if trig.suicide: - sp.add( trig.get( tag )) - else: - pp.add( trig.get( tag )) + if (self.ict is None or + trig.evaluation_offset_string is None or + (get_point_relative( + trig.evaluation_offset_string, tag) >= + self.ict)): + # i.c.t. can be None after a restart, if one + # is not specified in the suite definition. + + if trig.suicide: + sp.add( trig.get( tag )) + else: + pp.add( trig.get( tag )) sself.prerequisites.add_requisites( pp ) sself.suicide_prerequisites.add_requisites( sp ) @@ -270,7 +314,7 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, sself.startup = startup sself.submit_num = submit_num sself.exists=exists - sself.intercycle_offset = self.intercycle_offset + sself.intercycle_offsets = self.intercycle_offsets if self.cycling and startup: # adjust up to the first on-sequence cycle point @@ -282,10 +326,8 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, adjusted.append( adj ) if adjusted: sself.tag = min( adjusted ) - if sself.intercycle_offset is None: - sself.cleanup_cutoff = None - else: - sself.cleanup_cutoff = sself.tag + sself.intercycle_offset + sself.cleanup_cutoff = self.extract_cleanup_cutoff( + sself.tag, self.intercycle_offsets) sself.id = TaskID.get( sself.name, str(sself.tag) ) else: sself.tag = None @@ -294,10 +336,8 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, return else: sself.tag = start_point - if sself.intercycle_offset is None: - sself.cleanup_cutoff = None - else: - sself.cleanup_cutoff = sself.tag + sself.intercycle_offset + sself.cleanup_cutoff = self.extract_cleanup_cutoff( + sself.tag, self.intercycle_offsets) sself.id = TaskID.get( sself.name, str(sself.tag) ) sself.c_time = sself.tag diff --git a/lib/cylc/trigger.py b/lib/cylc/trigger.py index 65d4b2c9d5d..93b991465d7 100644 --- a/lib/cylc/trigger.py +++ b/lib/cylc/trigger.py @@ -17,7 +17,8 @@ #C: along with this program. If not, see . import cylc.TaskID -from cylc.cycling.loader import get_interval, get_interval_cls +from cylc.cycling.loader import ( + get_interval, get_interval_cls, get_point_relative) import re @@ -46,7 +47,7 @@ def __init__(self, name ): self.name = name self.msg = None self.intrinsic_offset = None - self.evaluation_offset = None + self.evaluation_offset_string = None self.cycle_point = None self.type = None self.cycling = False @@ -81,8 +82,8 @@ def set_type( self, type ): def set_cycle_point( self, cycle_point ): self.cycle_point = cycle_point - def set_offset( self, offset ): - self.evaluation_offset = get_interval( offset ) + def set_offset_string( self, offset_string ): + self.evaluation_offset_string = offset_string def get( self, ctime ): if self.msg: @@ -90,15 +91,17 @@ def get( self, ctime ): preq = self.msg if self.intrinsic_offset: ctime += self.intrinsic_offset - if self.evaluation_offset: - ctime -= self.evaluation_offset + if self.evaluation_offset_string: + ctime = get_point_relative( + self.evaluation_offset_string, ctime) if self.cycle_point: ctime = self.cycle_point - preq = re.sub( '\[\s*T\s*.*?\]', str(ctime), preq ) + preq = re.sub( '\[\s*[T\s*.*?\]', str(ctime), preq ) else: # implicit output - if self.evaluation_offset: - ctime -= self.evaluation_offset + if self.evaluation_offset_string: + ctime = get_point_relative( + self.evaluation_offset_string, ctime) if self.cycle_point: ctime = self.cycle_point preq = cylc.TaskID.get( self.name, str(ctime) ) + ' ' + self.type diff --git a/tests/cyclers/monthly_complex/suite.rc b/tests/cyclers/monthly_complex/suite.rc new file mode 100644 index 00000000000..b9e80efd580 --- /dev/null +++ b/tests/cyclers/monthly_complex/suite.rc @@ -0,0 +1,21 @@ +[cylc] + UTC mode = True + [[reference test]] + live mode suite timeout = PT2M +[scheduling] + initial cycle point = 20000331T0100Z + final cycle point = 2001 + [[dependencies]] + [[[R1/+P1M]]] + graph = "qux & flub" + [[[T00-P1M1D/P1M]]] + graph = "wibble => wobble" + [[[R10//P1D]]] + graph = "flub[^] => flob" + [[[02T06-P8DT6H/P1M]]] + graph = "foo[01T-P1W] & qux[^+P1M] => bar" + [[[01T-P1W/P1M]]] + graph = "foo => baz" +[runtime] + [[root]] + command scripting = true From 2981fc0ea48b2d0520557b58474477c3a1e2d1cb Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Wed, 23 Jul 2014 14:53:17 +0100 Subject: [PATCH 02/14] bug fix and testing suite update --- lib/cylc/config.py | 1 + lib/cylc/taskdef.py | 2 +- tests/cyclers/monthly_complex/suite.rc | 18 +++++++++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 86833e75ea2..de01483dfa7 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -881,6 +881,7 @@ def check_tasks( self ): # could cause a TypeError. raise SuiteConfigError('(inconsistent use of special tasks?)') except Exception, x: + raise raise SuiteConfigError( 'ERROR, failed to instantiate task %s: %s' % (name, x)) if not itask.tag: diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index deb29cc7c60..c1ae3dd00b0 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -129,7 +129,7 @@ def extract_cleanup_cutoff( self, my_point, offset_sequence_tuples ): break if target_point == my_point: matching_dependent_points.append(dependent_point) - print " cutoff point candidate:", cutoff_points[-1] + print " cutoff point candidate:", dependent_point dependent_point = sequence.get_next_point_on_sequence( dependent_point) if matching_dependent_points: diff --git a/tests/cyclers/monthly_complex/suite.rc b/tests/cyclers/monthly_complex/suite.rc index b9e80efd580..681679eacc8 100644 --- a/tests/cyclers/monthly_complex/suite.rc +++ b/tests/cyclers/monthly_complex/suite.rc @@ -6,15 +6,15 @@ initial cycle point = 20000331T0100Z final cycle point = 2001 [[dependencies]] - [[[R1/+P1M]]] - graph = "qux & flub" - [[[T00-P1M1D/P1M]]] - graph = "wibble => wobble" - [[[R10//P1D]]] - graph = "flub[^] => flob" - [[[02T06-P8DT6H/P1M]]] - graph = "foo[01T-P1W] & qux[^+P1M] => bar" - [[[01T-P1W/P1M]]] + # [[[R1/+P1M]]] + # graph = "qux & flub" + # [[[T00-P1M1D/P1M]]] + # graph = "wibble => wobble" + # [[[R10//P1D]]] + # graph = "flub[^+P1M] => flob" + [[[02T06-P8DT6H/P1W]]] + graph = "foo[01T+P1W] => bar" + [[[01T+P1W/P1M]]] graph = "foo => baz" [runtime] [[root]] From 54d5f009b2461bcaea31bd9e87a650c1126441d6 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Wed, 23 Jul 2014 16:39:36 +0100 Subject: [PATCH 03/14] add dynamic possible-future-trigger calculation --- lib/cylc/config.py | 1 - lib/cylc/task_pool.py | 38 +++++++++++++++++++++++++++++++------- lib/cylc/taskdef.py | 16 ++++++++-------- lib/cylc/trigger.py | 3 ++- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/lib/cylc/config.py b/lib/cylc/config.py index f8a1ae5e458..8b751e1208f 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -1329,7 +1329,6 @@ def generate_triggers( self, lexpression, left_nodes, right, seq, suicide ): ltaskdef.intercycle = True if lnode.offset_is_from_ict: - print "Get point relative", lnode.offset_string, l_taskdef.ict first_point = get_point_relative( lnode.offset_string, l_taskdef.ict) last_point = seq.get_stop_point() diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py index 485c00af8ff..0e02f2a9241 100644 --- a/lib/cylc/task_pool.py +++ b/lib/cylc/task_pool.py @@ -70,7 +70,8 @@ def __init__( self, suite, db, stop_point, config, pyro, log, run_mode, proc_poo self.db = db self.custom_runahead_limit = config.get_custom_runahead_limit() - self.minimum_runahead_limit = config.get_minimum_runahead_limit() + self.latest_prereq_point = None + self._prev_rh_stall_warning_base_point = None self.max_num_active_cycle_points = ( config.get_max_num_active_cycle_points()) self._prev_runahead_base_point = None @@ -210,16 +211,25 @@ def release_runahead_tasks( self ): # Calculate which tasks to release based on a maximum number of # active cycle points (active meaning non-finished tasks). latest_allowed_point = sorted(points)[:limit][-1] - if self.minimum_runahead_limit is not None: + if self.latest_prereq_point is not None: latest_allowed_point = max([ latest_allowed_point, - runahead_base_point + self.minimum_runahead_limit + self.latest_prereq_point ]) else: # Calculate which tasks to release based on a maximum duration # measured from the oldest non-finished task. latest_allowed_point = ( runahead_base_point + self.custom_runahead_limit) + if (self._prev_rh_stall_warning_point != runahead_base_point and + self.latest_prereq_point > latest_allowed_point): + offset = self.latest_prereq_point - runahead_base_point + self.log.warning( + 'custom runahead limit of %s is less than ' + 'future triggering offset %s: suite may stall.' % ( + self.custom_runahead_limit, offset) + ) + self._prev_rh_stall_warning_point = runahead_base_point for point, itask_id_map in self.runahead_pool.items(): if point <= latest_allowed_point: @@ -250,7 +260,8 @@ def release_runahead_task(self, itask): self.log.warning( '%s cannot be added (use --debug and see stderr)' % itask.id) return False - + if itask.max_future_prereq_point is not None: + self.set_latest_prereq_point() def remove( self, itask, reason=None ): try: @@ -284,6 +295,8 @@ def remove( self, itask, reason=None ): if reason: msg += " (" + reason + ")" itask.log( 'DEBUG', msg ) + if itask.max_future_prereq_point is not None: + self.set_latest_prereq_point() del itask @@ -437,10 +450,10 @@ def set_runahead( self, interval=None ): interval = get_interval(interval) if interval is None: # No limit - self.log.warning( "setting NO runahead limit" ) + self.log.warning( "setting NO custom runahead limit" ) self.custom_runahead_limit = None else: - self.log.info( "setting runahead limit to " + str(interval) ) + self.log.info( "setting custom runahead limit to %s" % interval ) self.custom_runahead_limit = interval self.release_runahead_tasks() @@ -463,12 +476,23 @@ def get_max_ctime( self ): return maxc + def set_latest_prereq_point(self): + """Calculate the latest required future trigger point.""" + max_point = None + for itask in self.get_tasks(): + if (itask.max_future_prereq_point is not None and + (max_point is None or + itask.max_future_prereq_point > max_point)): + max_point = itask.max_future_prereq_point + print "Latest prereq point", max_point + self.latest_prereq_point = max_point + + def reconfigure( self, config, stop_point ): self.reconfiguring = True self.custom_runahead_limit = config.get_custom_runahead_limit() - self.minimum_runahead_limit = config.get_minimum_runahead_limit() self.max_num_active_cycle_points = ( config.get_max_num_active_cycle_points()) self.config = config diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 2fcbd413540..7f0c0a5adf1 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -101,9 +101,8 @@ def add_sequence( self, sequence, is_implicit=False ): if is_implicit: self.implicit_sequences.append( sequence ) - def get_earliest_and_latest_dependent_points( self, my_point, - offset_sequence_tuples): - """Extract the min and max dependent cycle points at my_point.""" + def get_cleanup_cutoff_point( self, my_point, offset_sequence_tuples): + """Extract the max dependent cycle point for this point.""" print "Extract cleanup cutoff", self.name, my_point if not offset_sequence_tuples: print " not offset_seq_tuples: None" @@ -207,6 +206,7 @@ def get_task_class( self ): tclass.mean_total_elapsed_time = None tclass.intercycle = self.intercycle + tclass.max_future_prereq_point = None tclass.follow_on = self.follow_on_task tclass.namespace_hierarchy = self.namespace_hierarchy @@ -275,10 +275,10 @@ def tclass_add_prerequisites( sself, tag ): message, prereq_point = trig.get( tag ) if (prereq_point != tag and - (self.max_future_prereq_point is None or + (sself.max_future_prereq_point is None or prereq_point > - self.max_future_prereq_point)): - self.max_future_prereq_point = prereq_point + sself.max_future_prereq_point)): + sself.max_future_prereq_point = prereq_point if trig.suicide: sp.add( message ) @@ -335,7 +335,7 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, adjusted.append( adj ) if adjusted: sself.tag = min( adjusted ) - sself.cleanup_cutoff = self.get_earliest_and_latest_dependent_points( + sself.cleanup_cutoff = self.get_cleanup_cutoff_point( sself.tag, self.intercycle_offsets) sself.id = TaskID.get( sself.name, str(sself.tag) ) else: @@ -345,7 +345,7 @@ def tclass_init( sself, start_point, initial_state, stop_c_time=None, return else: sself.tag = start_point - sself.cleanup_cutoff = self.get_earliest_and_latest_dependent_points( + sself.cleanup_cutoff = self.get_cleanup_cutoff_point( sself.tag, self.intercycle_offsets) sself.id = TaskID.get( sself.name, str(sself.tag) ) diff --git a/lib/cylc/trigger.py b/lib/cylc/trigger.py index 93b991465d7..004c50b1663 100644 --- a/lib/cylc/trigger.py +++ b/lib/cylc/trigger.py @@ -86,6 +86,7 @@ def set_offset_string( self, offset_string ): self.evaluation_offset_string = offset_string def get( self, ctime ): + """Return a prerequisite string and the relevant point for ctime.""" if self.msg: # explicit internal output ... preq = self.msg @@ -105,4 +106,4 @@ def get( self, ctime ): if self.cycle_point: ctime = self.cycle_point preq = cylc.TaskID.get( self.name, str(ctime) ) + ' ' + self.type - return preq + return preq, ctime From 118db76b2e9a11d7e3b99bc6306dacd731446804 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Wed, 23 Jul 2014 17:07:33 +0100 Subject: [PATCH 04/14] #976: probably-working... --- lib/cylc/config.py | 2 +- lib/cylc/cycling/iso8601.py | 3 + lib/cylc/taskdef.py | 14 ++- tests/cyclers/monthly_complex/reference.log | 125 ++++++++++++++++++++ tests/cyclers/monthly_complex/suite.rc | 18 ++- 5 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 tests/cyclers/monthly_complex/reference.log diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 8b751e1208f..2074dfb08cd 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -1330,7 +1330,7 @@ def generate_triggers( self, lexpression, left_nodes, right, seq, suicide ): if lnode.offset_is_from_ict: first_point = get_point_relative( - lnode.offset_string, l_taskdef.ict) + lnode.offset_string, ltaskdef.ict) last_point = seq.get_stop_point() if last_point is None: # This dependency persists for the whole suite run. diff --git a/lib/cylc/cycling/iso8601.py b/lib/cylc/cycling/iso8601.py index cac6fb35dac..8f54d148c79 100755 --- a/lib/cylc/cycling/iso8601.py +++ b/lib/cylc/cycling/iso8601.py @@ -428,6 +428,9 @@ def __eq__(self, other): return True return False + def __str__(self): + return self.value + def convert_old_cycler_syntax(dep_section, only_detect_old=False, start_point=None): diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 7f0c0a5adf1..d544fba5438 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -31,7 +31,7 @@ import TaskID from task_output_logs import logfiles from parsec.OrderedDict import OrderedDict -from cycling.loader import get_interval_cls, get_point_relative +from cycling.loader import get_interval_cls, get_point_relative, get_interval class Error( Exception ): @@ -109,13 +109,13 @@ def get_cleanup_cutoff_point( self, my_point, offset_sequence_tuples): return None cutoff_points = [] for offset_string, sequence in offset_sequence_tuples: - print " offset_string, sequence", offset_string, sequence.value + print " offset_string, sequence", offset_string, str(sequence) if offset_string is None: # This indicates a dependency across the whole suite run. return None if sequence is None: cutoff_points.append( - get_point_relative(offset_string, my_point)) + my_point - get_interval(offset_string)) print " cutoff point candidate:", cutoff_points[-1] continue dependent_point = sequence.get_start_point() @@ -137,8 +137,12 @@ def get_cleanup_cutoff_point( self, my_point, offset_sequence_tuples): # Choose the largest of the dependent points. cutoff_points.append(matching_dependent_points[-1]) if cutoff_points: - print " cutoff point choice:", max(cutoff_points) - return max(cutoff_points) + max_cutoff_point = max(cutoff_points) + print " cutoff max:", max_cutoff_point + if max_cutoff_point < my_point: + print " too low, defaulting to:", my_point + return my_point + return max_cutoff_point print " no cutoff points:", None return None diff --git a/tests/cyclers/monthly_complex/reference.log b/tests/cyclers/monthly_complex/reference.log new file mode 100644 index 00000000000..e5299d6cbf6 --- /dev/null +++ b/tests/cyclers/monthly_complex/reference.log @@ -0,0 +1,125 @@ +2014-07-23T17:03:19+01 INFO - port:7766 +2014-07-23T17:03:19+01 INFO - Suite starting at 2014-07-23T17:03:19+01 +2014-07-23T17:03:19+01 INFO - Log event clock: real time +2014-07-23T17:03:19+01 INFO - Run mode: live +2014-07-23T17:03:19+01 INFO - Start tag: 20000331T0100Z +2014-07-23T17:03:19+01 INFO - Stop tag: 20010101T0000Z +2014-07-23T17:03:19+01 INFO - Cold Start 20000331T0100Z +2014-07-23T17:03:20+01 INFO - [flub.20000430T0100Z] -triggered off [] +2014-07-23T17:03:20+01 INFO - [foo.20000406T0000Z] -triggered off [] +2014-07-23T17:03:21+01 INFO - [flub.20000430T0100Z] -submit_method_id=22400 +2014-07-23T17:03:21+01 INFO - [flub.20000430T0100Z] -submission succeeded +2014-07-23T17:03:21+01 INFO - [foo.20000406T0000Z] -submit_method_id=22403 +2014-07-23T17:03:21+01 INFO - [foo.20000406T0000Z] -submission succeeded +2014-07-23T17:03:21+01 INFO - [flub.20000430T0100Z] -(current:submitted)> flub.20000430T0100Z started at 2014-07-23T16:03:21Z +2014-07-23T17:03:21+01 INFO - [foo.20000406T0000Z] -(current:submitted)> foo.20000406T0000Z started at 2014-07-23T16:03:21Z +2014-07-23T17:03:22+01 INFO - [flub.20000430T0100Z] -(current:running)> flub.20000430T0100Z succeeded at 2014-07-23T16:03:21Z +2014-07-23T17:03:22+01 INFO - [foo.20000406T0000Z] -(current:running)> foo.20000406T0000Z succeeded at 2014-07-23T16:03:21Z +2014-07-23T17:03:23+01 INFO - [flob.20000331T0100Z] -triggered off ['flub.20000430T0100Z'] +2014-07-23T17:03:24+01 INFO - [flob.20000331T0100Z] -submit_method_id=22609 +2014-07-23T17:03:24+01 INFO - [flob.20000331T0100Z] -submission succeeded +2014-07-23T17:03:24+01 INFO - [flob.20000331T0100Z] -(current:submitted)> flob.20000331T0100Z started at 2014-07-23T16:03:24Z +2014-07-23T17:03:25+01 INFO - [flob.20000331T0100Z] -(current:running)> flob.20000331T0100Z succeeded at 2014-07-23T16:03:24Z +2014-07-23T17:03:26+01 INFO - [flob.20000401T0100Z] -triggered off ['flob.20000331T0100Z', 'flub.20000430T0100Z'] +2014-07-23T17:03:27+01 INFO - [flob.20000401T0100Z] -submit_method_id=22713 +2014-07-23T17:03:27+01 INFO - [flob.20000401T0100Z] -submission succeeded +2014-07-23T17:03:27+01 INFO - [flob.20000401T0100Z] -(current:submitted)> flob.20000401T0100Z started at 2014-07-23T16:03:27Z +2014-07-23T17:03:27+01 INFO - [flob.20000401T0100Z] -(current:running)> flob.20000401T0100Z succeeded at 2014-07-23T16:03:27Z +2014-07-23T17:03:28+01 INFO - [flob.20000402T0100Z] -triggered off ['flob.20000401T0100Z', 'flub.20000430T0100Z'] +2014-07-23T17:03:29+01 INFO - [flob.20000402T0100Z] -submit_method_id=22817 +2014-07-23T17:03:29+01 INFO - [flob.20000402T0100Z] -submission succeeded +2014-07-23T17:03:29+01 INFO - [flob.20000402T0100Z] -(current:submitted)> flob.20000402T0100Z started at 2014-07-23T16:03:29Z +2014-07-23T17:03:29+01 INFO - [flob.20000402T0100Z] -(current:running)> flob.20000402T0100Z succeeded at 2014-07-23T16:03:29Z +2014-07-23T17:03:30+01 INFO - [flob.20000403T0100Z] -triggered off ['flob.20000402T0100Z', 'flub.20000430T0100Z'] +2014-07-23T17:03:31+01 INFO - [flob.20000403T0100Z] -submit_method_id=22921 +2014-07-23T17:03:31+01 INFO - [flob.20000403T0100Z] -submission succeeded +2014-07-23T17:03:32+01 INFO - [flob.20000403T0100Z] -(current:submitted)> flob.20000403T0100Z started at 2014-07-23T16:03:31Z +2014-07-23T17:03:32+01 INFO - [flob.20000403T0100Z] -(current:running)> flob.20000403T0100Z succeeded at 2014-07-23T16:03:31Z +2014-07-23T17:03:33+01 INFO - [foo.20000506T0000Z] -triggered off [] +2014-07-23T17:03:34+01 INFO - [foo.20000506T0000Z] -submit_method_id=23025 +2014-07-23T17:03:34+01 INFO - [foo.20000506T0000Z] -submission succeeded +2014-07-23T17:03:34+01 INFO - [foo.20000506T0000Z] -(current:submitted)> foo.20000506T0000Z started at 2014-07-23T16:03:33Z +2014-07-23T17:03:34+01 INFO - [foo.20000506T0000Z] -(current:running)> foo.20000506T0000Z succeeded at 2014-07-23T16:03:33Z +2014-07-23T17:03:35+01 INFO - [bar.20000508T1200Z] -triggered off ['foo.20000506T0000Z'] +2014-07-23T17:03:35+01 INFO - [foo.20000606T0000Z] -triggered off [] +2014-07-23T17:03:36+01 INFO - [bar.20000508T1200Z] -submit_method_id=23129 +2014-07-23T17:03:36+01 INFO - [bar.20000508T1200Z] -submission succeeded +2014-07-23T17:03:36+01 INFO - [foo.20000606T0000Z] -submit_method_id=23132 +2014-07-23T17:03:36+01 INFO - [foo.20000606T0000Z] -submission succeeded +2014-07-23T17:03:36+01 INFO - [bar.20000508T1200Z] -(current:submitted)> bar.20000508T1200Z started at 2014-07-23T16:03:35Z +2014-07-23T17:03:36+01 INFO - [bar.20000508T1200Z] -(current:running)> bar.20000508T1200Z succeeded at 2014-07-23T16:03:35Z +2014-07-23T17:03:36+01 INFO - [foo.20000606T0000Z] -(current:submitted)> foo.20000606T0000Z started at 2014-07-23T16:03:35Z +2014-07-23T17:03:36+01 INFO - [foo.20000606T0000Z] -(current:running)> foo.20000606T0000Z succeeded at 2014-07-23T16:03:35Z +2014-07-23T17:03:37+01 INFO - [bar.20000605T1200Z] -triggered off ['foo.20000606T0000Z'] +2014-07-23T17:03:38+01 INFO - [bar.20000605T1200Z] -submit_method_id=23337 +2014-07-23T17:03:38+01 INFO - [bar.20000605T1200Z] -submission succeeded +2014-07-23T17:03:38+01 INFO - [bar.20000605T1200Z] -(current:submitted)> bar.20000605T1200Z started at 2014-07-23T16:03:37Z +2014-07-23T17:03:38+01 INFO - [bar.20000605T1200Z] -(current:running)> bar.20000605T1200Z succeeded at 2014-07-23T16:03:37Z +2014-07-23T17:03:39+01 INFO - [foo.20000706T0000Z] -triggered off [] +2014-07-23T17:03:40+01 INFO - [foo.20000706T0000Z] -submit_method_id=23441 +2014-07-23T17:03:40+01 INFO - [foo.20000706T0000Z] -submission succeeded +2014-07-23T17:03:40+01 INFO - [foo.20000706T0000Z] -(current:submitted)> foo.20000706T0000Z started at 2014-07-23T16:03:39Z +2014-07-23T17:03:40+01 INFO - [foo.20000706T0000Z] -(current:running)> foo.20000706T0000Z succeeded at 2014-07-23T16:03:39Z +2014-07-23T17:03:41+01 INFO - [bar.20000703T1200Z] -triggered off ['foo.20000706T0000Z'] +2014-07-23T17:03:42+01 INFO - [bar.20000703T1200Z] -submit_method_id=23545 +2014-07-23T17:03:42+01 INFO - [bar.20000703T1200Z] -submission succeeded +2014-07-23T17:03:42+01 INFO - [bar.20000703T1200Z] -(current:submitted)> bar.20000703T1200Z started at 2014-07-23T16:03:41Z +2014-07-23T17:03:42+01 INFO - [bar.20000703T1200Z] -(current:running)> bar.20000703T1200Z succeeded at 2014-07-23T16:03:41Z +2014-07-23T17:03:43+01 INFO - [foo.20000806T0000Z] -triggered off [] +2014-07-23T17:03:44+01 INFO - [foo.20000806T0000Z] -submit_method_id=23650 +2014-07-23T17:03:44+01 INFO - [foo.20000806T0000Z] -submission succeeded +2014-07-23T17:03:44+01 INFO - [foo.20000806T0000Z] -(current:submitted)> foo.20000806T0000Z started at 2014-07-23T16:03:43Z +2014-07-23T17:03:45+01 INFO - [foo.20000806T0000Z] -(current:running)> foo.20000806T0000Z succeeded at 2014-07-23T16:03:44Z +2014-07-23T17:03:46+01 INFO - [bar.20000731T1200Z] -triggered off ['foo.20000806T0000Z'] +2014-07-23T17:03:47+01 INFO - [bar.20000731T1200Z] -submit_method_id=23754 +2014-07-23T17:03:47+01 INFO - [bar.20000731T1200Z] -submission succeeded +2014-07-23T17:03:47+01 INFO - [bar.20000731T1200Z] -(current:submitted)> bar.20000731T1200Z started at 2014-07-23T16:03:46Z +2014-07-23T17:03:47+01 INFO - [bar.20000731T1200Z] -(current:running)> bar.20000731T1200Z succeeded at 2014-07-23T16:03:46Z +2014-07-23T17:03:48+01 INFO - [foo.20000906T0000Z] -triggered off [] +2014-07-23T17:03:49+01 INFO - [foo.20000906T0000Z] -submit_method_id=23858 +2014-07-23T17:03:49+01 INFO - [foo.20000906T0000Z] -submission succeeded +2014-07-23T17:03:49+01 INFO - [foo.20000906T0000Z] -(current:submitted)> foo.20000906T0000Z started at 2014-07-23T16:03:48Z +2014-07-23T17:03:49+01 INFO - [foo.20000906T0000Z] -(current:running)> foo.20000906T0000Z succeeded at 2014-07-23T16:03:48Z +2014-07-23T17:03:50+01 INFO - [bar.20000828T1200Z] -triggered off ['foo.20000906T0000Z'] +2014-07-23T17:03:51+01 INFO - [bar.20000828T1200Z] -submit_method_id=23962 +2014-07-23T17:03:51+01 INFO - [bar.20000828T1200Z] -submission succeeded +2014-07-23T17:03:51+01 INFO - [bar.20000828T1200Z] -(current:submitted)> bar.20000828T1200Z started at 2014-07-23T16:03:50Z +2014-07-23T17:03:52+01 INFO - [bar.20000828T1200Z] -(current:running)> bar.20000828T1200Z succeeded at 2014-07-23T16:03:51Z +2014-07-23T17:03:53+01 INFO - [foo.20001006T0000Z] -triggered off [] +2014-07-23T17:03:54+01 INFO - [foo.20001006T0000Z] -submit_method_id=24066 +2014-07-23T17:03:54+01 INFO - [foo.20001006T0000Z] -submission succeeded +2014-07-23T17:03:54+01 INFO - [foo.20001006T0000Z] -(current:submitted)> foo.20001006T0000Z started at 2014-07-23T16:03:53Z +2014-07-23T17:03:54+01 INFO - [foo.20001006T0000Z] -(current:running)> foo.20001006T0000Z succeeded at 2014-07-23T16:03:54Z +2014-07-23T17:03:55+01 INFO - [bar.20000925T1200Z] -triggered off ['foo.20001006T0000Z'] +2014-07-23T17:03:56+01 INFO - [bar.20000925T1200Z] -submit_method_id=24170 +2014-07-23T17:03:56+01 INFO - [bar.20000925T1200Z] -submission succeeded +2014-07-23T17:03:56+01 INFO - [bar.20000925T1200Z] -(current:submitted)> bar.20000925T1200Z started at 2014-07-23T16:03:55Z +2014-07-23T17:03:56+01 INFO - [bar.20000925T1200Z] -(current:running)> bar.20000925T1200Z succeeded at 2014-07-23T16:03:56Z +2014-07-23T17:03:57+01 INFO - [foo.20001106T0000Z] -triggered off [] +2014-07-23T17:03:58+01 INFO - [foo.20001106T0000Z] -submit_method_id=24274 +2014-07-23T17:03:58+01 INFO - [foo.20001106T0000Z] -submission succeeded +2014-07-23T17:03:58+01 INFO - [foo.20001106T0000Z] -(current:submitted)> foo.20001106T0000Z started at 2014-07-23T16:03:57Z +2014-07-23T17:03:58+01 INFO - [foo.20001106T0000Z] -(current:running)> foo.20001106T0000Z succeeded at 2014-07-23T16:03:58Z +2014-07-23T17:03:59+01 INFO - [bar.20001023T1200Z] -triggered off ['foo.20001106T0000Z'] +2014-07-23T17:04:00+01 INFO - [bar.20001023T1200Z] -submit_method_id=24378 +2014-07-23T17:04:00+01 INFO - [bar.20001023T1200Z] -submission succeeded +2014-07-23T17:04:00+01 INFO - [bar.20001023T1200Z] -(current:submitted)> bar.20001023T1200Z started at 2014-07-23T16:04:00Z +2014-07-23T17:04:00+01 INFO - [bar.20001023T1200Z] -(current:running)> bar.20001023T1200Z succeeded at 2014-07-23T16:04:00Z +2014-07-23T17:04:01+01 INFO - [foo.20001206T0000Z] -triggered off [] +2014-07-23T17:04:02+01 INFO - [foo.20001206T0000Z] -submit_method_id=24482 +2014-07-23T17:04:02+01 INFO - [foo.20001206T0000Z] -submission succeeded +2014-07-23T17:04:02+01 INFO - [foo.20010106T0000Z] -holding (beyond suite stop point) 20010101T0000Z +2014-07-23T17:04:02+01 INFO - [foo.20001206T0000Z] -(current:submitted)> foo.20001206T0000Z started at 2014-07-23T16:04:02Z +2014-07-23T17:04:02+01 INFO - [foo.20001206T0000Z] -(current:running)> foo.20001206T0000Z succeeded at 2014-07-23T16:04:02Z +2014-07-23T17:04:03+01 INFO - [bar.20001120T1200Z] -triggered off ['foo.20001206T0000Z'] +2014-07-23T17:04:04+01 INFO - [bar.20001120T1200Z] -submit_method_id=24586 +2014-07-23T17:04:04+01 INFO - [bar.20001120T1200Z] -submission succeeded +2014-07-23T17:04:04+01 INFO - [bar.20001120T1200Z] -(current:submitted)> bar.20001120T1200Z started at 2014-07-23T16:04:04Z +2014-07-23T17:04:04+01 INFO - [bar.20001120T1200Z] -(current:running)> bar.20001120T1200Z succeeded at 2014-07-23T16:04:04Z +2014-07-23T17:04:05+01 INFO - [bar.20001218T1200Z] -triggered off ['foo.20001206T0000Z'] +2014-07-23T17:04:06+01 INFO - [bar.20001218T1200Z] -submit_method_id=24690 +2014-07-23T17:04:06+01 INFO - [bar.20001218T1200Z] -submission succeeded +2014-07-23T17:04:06+01 INFO - [bar.20010115T1200Z] -holding (beyond suite stop point) 20010101T0000Z +2014-07-23T17:04:06+01 INFO - [bar.20001218T1200Z] -(current:submitted)> bar.20001218T1200Z started at 2014-07-23T16:04:06Z +2014-07-23T17:04:06+01 INFO - [bar.20001218T1200Z] -(current:running)> bar.20001218T1200Z succeeded at 2014-07-23T16:04:06Z +2014-07-23T17:04:07+01 INFO - Suite shutting down at 2014-07-23T17:04:07+01 diff --git a/tests/cyclers/monthly_complex/suite.rc b/tests/cyclers/monthly_complex/suite.rc index 681679eacc8..57cae4a39da 100644 --- a/tests/cyclers/monthly_complex/suite.rc +++ b/tests/cyclers/monthly_complex/suite.rc @@ -6,16 +6,14 @@ initial cycle point = 20000331T0100Z final cycle point = 2001 [[dependencies]] - # [[[R1/+P1M]]] - # graph = "qux & flub" - # [[[T00-P1M1D/P1M]]] - # graph = "wibble => wobble" - # [[[R10//P1D]]] - # graph = "flub[^+P1M] => flob" - [[[02T06-P8DT6H/P1W]]] - graph = "foo[01T+P1W] => bar" - [[[01T+P1W/P1M]]] - graph = "foo => baz" + [[[20T-P2W/P1M]]] + graph = "foo" + [[[30T06+P8DT6H/P3W]]] + graph = "foo[20T-P2W] => bar" + [[[R1/+P1M]]] + graph = "flub" + [[[R4//P1D]]] + graph = "flub[^+P1M] & flob[-P1D] => flob" [runtime] [[root]] command scripting = true From b47b35bf1b4b3d43a796df1acd92942adf9a88c3 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Wed, 23 Jul 2014 17:20:37 +0100 Subject: [PATCH 05/14] fix bugs vs test-battery --- lib/cylc/task_pool.py | 22 ++++++++++++---------- lib/cylc/taskdef.py | 12 ++++++++---- lib/cylc/trigger.py | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py index 0e02f2a9241..c1d8bda7801 100644 --- a/lib/cylc/task_pool.py +++ b/lib/cylc/task_pool.py @@ -71,7 +71,7 @@ def __init__( self, suite, db, stop_point, config, pyro, log, run_mode, proc_poo self.custom_runahead_limit = config.get_custom_runahead_limit() self.latest_prereq_point = None - self._prev_rh_stall_warning_base_point = None + self._prev_runahead_base_point = None self.max_num_active_cycle_points = ( config.get_max_num_active_cycle_points()) self._prev_runahead_base_point = None @@ -221,15 +221,17 @@ def release_runahead_tasks( self ): # measured from the oldest non-finished task. latest_allowed_point = ( runahead_base_point + self.custom_runahead_limit) - if (self._prev_rh_stall_warning_point != runahead_base_point and - self.latest_prereq_point > latest_allowed_point): - offset = self.latest_prereq_point - runahead_base_point - self.log.warning( - 'custom runahead limit of %s is less than ' - 'future triggering offset %s: suite may stall.' % ( - self.custom_runahead_limit, offset) - ) - self._prev_rh_stall_warning_point = runahead_base_point + + if (self._prev_runahead_base_point is None or + self._prev_runahead_base_point != runahead_base_point): + if self.latest_prereq_point > latest_allowed_point: + offset = self.latest_prereq_point - runahead_base_point + self.log.warning( + 'custom runahead limit of %s is less than ' + 'future triggering offset %s: suite may stall.' % ( + self.custom_runahead_limit, offset) + ) + self._prev_runahead_base_point = runahead_base_point for point, itask_id_map in self.runahead_pool.items(): if point <= latest_allowed_point: diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index d544fba5438..ec8df42bd8f 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -302,13 +302,17 @@ def tclass_add_prerequisites( sself, tag ): cp = conditional_prerequisites( sself.id, self.ict ) for label in ctrig: trig = ctrig[label] - if self.ict is not None and trig.evaluation_offset is not None: + if (self.ict is not None and + trig.evaluation_offset_string is not None): is_less_than_ict = ( - tag - trig.evaluation_offset < self.ict) - cp.add( trig.get( tag ), label, + get_point_relative( + trig.evaluation_offset_string, tag) < + self.ict + ) + cp.add( trig.get( tag )[0], label, is_less_than_ict) else: - cp.add( trig.get( tag ), label ) + cp.add( trig.get( tag )[0], label ) cp.set_condition( exp ) if ctrig[foo].suicide: sself.suicide_prerequisites.add_requisites( cp ) diff --git a/lib/cylc/trigger.py b/lib/cylc/trigger.py index 004c50b1663..9468ee765ac 100644 --- a/lib/cylc/trigger.py +++ b/lib/cylc/trigger.py @@ -97,7 +97,7 @@ def get( self, ctime ): self.evaluation_offset_string, ctime) if self.cycle_point: ctime = self.cycle_point - preq = re.sub( '\[\s*[T\s*.*?\]', str(ctime), preq ) + preq = re.sub( '\[\s*T\s*.*?\]', str(ctime), preq ) else: # implicit output if self.evaluation_offset_string: From dd1d2d54ad03a2f249541c8c93231a5b3081561a Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Mon, 28 Jul 2014 15:55:19 +0100 Subject: [PATCH 06/14] #976: tidy up offset bugs --- lib/cylc/config.py | 2 +- lib/cylc/cycling/iso8601.py | 4 ++-- lib/cylc/graphnode.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 2074dfb08cd..9def93ae813 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -1286,7 +1286,7 @@ def generate_taskdefs( self, line, left_nodes, right, ttype, section, seq, self.cfg['scheduling']['final cycle point'] ) seq_offset.set_offset( - -get_interval(offset_string)) + get_interval(offset_string)) offset_seq_map[offset_string] = seq_offset self.taskdefs[name].add_sequence( seq_offset, is_implicit=True) diff --git a/lib/cylc/cycling/iso8601.py b/lib/cylc/cycling/iso8601.py index 8f54d148c79..620856971d8 100755 --- a/lib/cylc/cycling/iso8601.py +++ b/lib/cylc/cycling/iso8601.py @@ -311,9 +311,9 @@ def get_offset(self): def set_offset(self, offset): """Alter state to offset the entire sequence.""" if self.recurrence.start_point is not None: - self.recurrence.start_point -= interval_parse(str(offset)) + self.recurrence.start_point += interval_parse(str(offset)) if self.recurrence.end_point is not None: - self.recurrence.end_point -= interval_parse(str(offset)) + self.recurrence.end_point += interval_parse(str(offset)) self._cached_first_point_values = {} self._cached_next_point_values = {} self._cached_valid_point_booleans = {} diff --git a/lib/cylc/graphnode.py b/lib/cylc/graphnode.py index 6848eb5e827..d1609d46970 100755 --- a/lib/cylc/graphnode.py +++ b/lib/cylc/graphnode.py @@ -123,6 +123,7 @@ def __init__( self, node, base_interval=None ): is_prev_cycling_format = True # node looks like foo[T-6], foo[T-12]:fail... name, sign, offset_string, outp = m.groups() + offset_string = sign + offset_string prev_format = True if outp: From b044a60dfbe6bccdaa9ff21de6cdd5de2db06583 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Tue, 29 Jul 2014 10:39:22 +0100 Subject: [PATCH 07/14] #976: fix integer cycling --- lib/cylc/cycling/integer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/cycling/integer.py b/lib/cylc/cycling/integer.py index 86b68149be1..c05de7d9470 100755 --- a/lib/cylc/cycling/integer.py +++ b/lib/cylc/cycling/integer.py @@ -401,7 +401,7 @@ def init_from_cfg(cfg): def get_point_relative(offset_string, base_point): """Create a point from offset_string applied to base_point.""" # TODO ISO: needs to be done somehow? Is it meaningful? - raise NotImplementedError() + return base_point + IntegerInterval(offset_string) if __name__ == '__main__': From e785a4f7270b7bab067b48cfe8cc99aff9e56bb4 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Tue, 29 Jul 2014 16:25:15 +0100 Subject: [PATCH 08/14] fix update bug --- lib/cylc/taskdef.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index ab3b8472f5a..4491a50d84e 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -355,7 +355,7 @@ def tclass_init( sself, start_point, initial_state, stop_point=None, else: sself.point = start_point sself.cleanup_cutoff = self.get_cleanup_cutoff_point( - sself.tag, self.intercycle_offsets) + sself.point, self.intercycle_offsets) sself.id = TaskID.get( sself.name, str(sself.point) ) if 'clocktriggered' in self.modifiers: From 234d271d3a3e78c49ae8bbdbae7c800828cea338 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Tue, 29 Jul 2014 16:25:25 +0100 Subject: [PATCH 09/14] rephrase runahead tests --- tests/runahead/01-check-default-simple.t | 15 ++++++--------- tests/runahead/02-check-default-complex.t | 15 ++++++--------- tests/runahead/03-check-default-future.t | 15 ++++++--------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/tests/runahead/01-check-default-simple.t b/tests/runahead/01-check-default-simple.t index 0a76bc5eacd..54632c1457e 100644 --- a/tests/runahead/01-check-default-simple.t +++ b/tests/runahead/01-check-default-simple.t @@ -18,7 +18,7 @@ #C: Test default runahead limit behaviour is still the same . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 4 +set_test_number 5 #------------------------------------------------------------------------------- install_suite $TEST_NAME_BASE default-simple #------------------------------------------------------------------------------- @@ -28,15 +28,12 @@ run_ok $TEST_NAME cylc validate -v $SUITE_NAME TEST_NAME=$TEST_NAME_BASE-run run_fail $TEST_NAME cylc run --debug $SUITE_NAME #------------------------------------------------------------------------------- -TEST_NAME=$TEST_NAME_BASE-check-fail +TEST_NAME=$TEST_NAME_BASE-max-cycle DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db -RUNAHEAD=$(sqlite3 $DB "select max(cycle) from task_states") -# manual comparison for the test -if [[ "$RUNAHEAD" == "20100101T1200Z" ]]; then - ok $TEST_NAME -else - fail $TEST_NAME -fi +run_ok $TEST_NAME sqlite3 $DB "select max(cycle) from task_states" +cmp_ok "$TEST_NAME.stdout" <<'__OUT__' +20100101T1200Z +__OUT__ #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-check-timeout LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log diff --git a/tests/runahead/02-check-default-complex.t b/tests/runahead/02-check-default-complex.t index 6bb36a69cef..1449fd7e321 100644 --- a/tests/runahead/02-check-default-complex.t +++ b/tests/runahead/02-check-default-complex.t @@ -18,7 +18,7 @@ #C: Test default runahead limit behaviour is still the same . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 4 +set_test_number 5 #------------------------------------------------------------------------------- install_suite $TEST_NAME_BASE default-complex #------------------------------------------------------------------------------- @@ -28,15 +28,12 @@ run_ok $TEST_NAME cylc validate -v $SUITE_NAME TEST_NAME=$TEST_NAME_BASE-run run_fail $TEST_NAME cylc run --debug $SUITE_NAME #------------------------------------------------------------------------------- -TEST_NAME=$TEST_NAME_BASE-check-fail +TEST_NAME=$TEST_NAME_BASE-max-cycle DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db -RUNAHEAD=$(sqlite3 $DB "select max(cycle) from task_states") -# manual comparison for the test -if [[ "$RUNAHEAD" == "20100101T0500Z" ]]; then - ok $TEST_NAME -else - fail $TEST_NAME -fi +run_ok $TEST_NAME sqlite3 $DB "select max(cycle) from task_states" +cmp_ok "$TEST_NAME.stdout" <<'__OUT__' +20100101T0500Z +__OUT__ #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-check-timeout LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log diff --git a/tests/runahead/03-check-default-future.t b/tests/runahead/03-check-default-future.t index 54177996c0a..d45dac8507a 100644 --- a/tests/runahead/03-check-default-future.t +++ b/tests/runahead/03-check-default-future.t @@ -18,7 +18,7 @@ # Test default runahead limit behaviour is still the same . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 4 +set_test_number 5 #------------------------------------------------------------------------------- install_suite $TEST_NAME_BASE default-future #------------------------------------------------------------------------------- @@ -28,15 +28,12 @@ run_ok $TEST_NAME cylc validate -v $SUITE_NAME TEST_NAME=$TEST_NAME_BASE-run run_fail $TEST_NAME cylc run --debug $SUITE_NAME #------------------------------------------------------------------------------- -TEST_NAME=$TEST_NAME_BASE-check-fail +TEST_NAME=$TEST_NAME_BASE-max-cycle DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db -RUNAHEAD=$(sqlite3 $DB "select max(cycle) from task_states") -# manual comparison for the test -if [[ "$RUNAHEAD" == "20100101T0600Z" ]]; then - ok $TEST_NAME -else - fail $TEST_NAME -fi +run_ok $TEST_NAME sqlite3 $DB "select max(cycle) from task_states" +cmp_ok "$TEST_NAME.stdout" <<'__OUT__' +20100101T0600Z +__OUT__ #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-check-timeout LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log From a00558815d4a1349a95b97c51a925eedfae75544 Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Tue, 29 Jul 2014 17:16:25 +0100 Subject: [PATCH 10/14] #976: tweak future trigger runahead problem --- lib/cylc/task_pool.py | 46 +++++++++++----------- lib/cylc/taskdef.py | 15 +++---- tests/runahead/03-check-default-future.t | 8 ++-- tests/runahead/05-check-default-future-2.t | 44 +++++++++++++++++++++ tests/runahead/default-future/suite.rc | 12 +++--- 5 files changed, 87 insertions(+), 38 deletions(-) create mode 100644 tests/runahead/05-check-default-future-2.t diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py index 73d147056b6..16147accaa4 100644 --- a/lib/cylc/task_pool.py +++ b/lib/cylc/task_pool.py @@ -70,7 +70,7 @@ def __init__( self, suite, db, stop_point, config, pyro, log, run_mode, proc_poo self.db = db self.custom_runahead_limit = config.get_custom_runahead_limit() - self.latest_prereq_point = None + self.max_future_offset = None self._prev_runahead_base_point = None self.max_num_active_cycle_points = ( config.get_max_num_active_cycle_points()) @@ -211,11 +211,9 @@ def release_runahead_tasks( self ): # Calculate which tasks to release based on a maximum number of # active cycle points (active meaning non-finished tasks). latest_allowed_point = sorted(points)[:limit][-1] - if self.latest_prereq_point is not None: - latest_allowed_point = max([ - latest_allowed_point, - self.latest_prereq_point - ]) + if self.max_future_offset is not None: + # For the first N points, release their future trigger tasks. + latest_allowed_point += self.max_future_offset else: # Calculate which tasks to release based on a maximum duration # measured from the oldest non-finished task. @@ -224,12 +222,13 @@ def release_runahead_tasks( self ): if (self._prev_runahead_base_point is None or self._prev_runahead_base_point != runahead_base_point): - if self.latest_prereq_point > latest_allowed_point: - offset = self.latest_prereq_point - runahead_base_point + if self.custom_runahead_limit < self.max_future_offset: self.log.warning( - 'custom runahead limit of %s is less than ' + 'custom runahead limit of %s is less than ' + 'future triggering offset %s: suite may stall.' % ( - self.custom_runahead_limit, offset) + self.custom_runahead_limit, + self.max_future_offset + ) ) self._prev_runahead_base_point = runahead_base_point @@ -262,8 +261,9 @@ def release_runahead_task(self, itask): self.log.warning( '%s cannot be added (use --debug and see stderr)' % itask.id) return False - if itask.max_future_prereq_point is not None: - self.set_latest_prereq_point() + print "RELEASE RUNAHEAD", itask.id, itask.max_future_prereq_offset + if itask.max_future_prereq_offset is not None: + self.set_max_future_offset() def remove( self, itask, reason=None ): @@ -298,8 +298,8 @@ def remove( self, itask, reason=None ): if reason: msg += " (" + reason + ")" itask.log( 'DEBUG', msg ) - if itask.max_future_prereq_point is not None: - self.set_latest_prereq_point() + if itask.max_future_prereq_offset is not None: + self.set_max_future_offset() del itask @@ -479,16 +479,16 @@ def get_max_point( self ): return maxc - def set_latest_prereq_point(self): - """Calculate the latest required future trigger point.""" - max_point = None + def set_max_future_offset(self): + """Calculate the latest required future trigger offset.""" + max_offset = None for itask in self.get_tasks(): - if (itask.max_future_prereq_point is not None and - (max_point is None or - itask.max_future_prereq_point > max_point)): - max_point = itask.max_future_prereq_point - print "Latest prereq point", max_point - self.latest_prereq_point = max_point + if (itask.max_future_prereq_offset is not None and + (max_offset is None or + itask.max_future_prereq_offset > max_offset)): + max_offset = itask.max_future_prereq_offset + print "Max future offset", max_offset + self.max_future_offset = max_offset def reconfigure( self, config, stop_point ): diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 4491a50d84e..b40b75c5edc 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -64,7 +64,7 @@ def __init__( self, name, rtcfg, run_mode, ict ): # some defaults self.intercycle = False - self.max_future_prereq_point = None + self.max_future_prereq_offset = None self.intercycle_offsets = [] self.sequential = False self.cycling = False @@ -210,7 +210,7 @@ def get_task_class( self ): tclass.mean_total_elapsed_time = None tclass.intercycle = self.intercycle - tclass.max_future_prereq_point = None + tclass.max_future_prereq_offset = None tclass.follow_on = self.follow_on_task tclass.namespace_hierarchy = self.namespace_hierarchy @@ -278,11 +278,12 @@ def tclass_add_prerequisites( sself, point ): # is not specified in the suite definition. message, prereq_point = trig.get( point ) - if (prereq_point != point and - (sself.max_future_prereq_point is None or - prereq_point > - sself.max_future_prereq_point)): - sself.max_future_prereq_point = prereq_point + prereq_offset = prereq_point - point + if (prereq_offset > get_interval_cls().get_null() and + (sself.max_future_prereq_offset is None or + prereq_offset > + sself.max_future_prereq_offset)): + sself.max_future_prereq_offset = prereq_offset if trig.suicide: sp.add( message ) diff --git a/tests/runahead/03-check-default-future.t b/tests/runahead/03-check-default-future.t index d45dac8507a..ecebc403c00 100644 --- a/tests/runahead/03-check-default-future.t +++ b/tests/runahead/03-check-default-future.t @@ -23,16 +23,18 @@ set_test_number 5 install_suite $TEST_NAME_BASE default-future #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate -run_ok $TEST_NAME cylc validate -v $SUITE_NAME +run_ok $TEST_NAME cylc validate -v --set=FUTURE_TRIGGER_START_POINT=T04 \ + $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-run -run_fail $TEST_NAME cylc run --debug $SUITE_NAME +run_fail $TEST_NAME cylc run --debug --set=FUTURE_TRIGGER_START_POINT=T04 \ + $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-max-cycle DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db run_ok $TEST_NAME sqlite3 $DB "select max(cycle) from task_states" cmp_ok "$TEST_NAME.stdout" <<'__OUT__' -20100101T0600Z +20100101T0200Z __OUT__ #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-check-timeout diff --git a/tests/runahead/05-check-default-future-2.t b/tests/runahead/05-check-default-future-2.t new file mode 100644 index 00000000000..7f726f90410 --- /dev/null +++ b/tests/runahead/05-check-default-future-2.t @@ -0,0 +1,44 @@ +#!/bin/bash +#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE. +#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA +#C: +#C: This program is free software: you can redistribute it and/or modify +#C: it under the terms of the GNU General Public License as published by +#C: the Free Software Foundation, either version 3 of the License, or +#C: (at your option) any later version. +#C: +#C: This program is distributed in the hope that it will be useful, +#C: but WITHOUT ANY WARRANTY; without even the implied warranty of +#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#C: GNU General Public License for more details. +#C: +#C: You should have received a copy of the GNU General Public License +#C: along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test default runahead limit behaviour is still the same +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +set_test_number 5 +#------------------------------------------------------------------------------- +install_suite $TEST_NAME_BASE default-future +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-validate +run_ok $TEST_NAME cylc validate -v --set=FUTURE_TRIGGER_START_POINT=T02 \ + $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-run +run_fail $TEST_NAME cylc run --debug --set=FUTURE_TRIGGER_START_POINT=T02 \ + $SUITE_NAME +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-max-cycle +DB=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/cylc-suite.db +run_ok $TEST_NAME sqlite3 $DB "select max(cycle) from task_states" +cmp_ok "$TEST_NAME.stdout" <<'__OUT__' +20100101T0800Z +__OUT__ +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-check-timeout +LOG=$(cylc get-global-config --print-run-dir)/$SUITE_NAME/log/suite/log +run_ok $TEST_NAME grep 'suite timed out after' $LOG +#------------------------------------------------------------------------------- +purge_suite $SUITE_NAME diff --git a/tests/runahead/default-future/suite.rc b/tests/runahead/default-future/suite.rc index d10cfa5c413..a7fdf24a44f 100644 --- a/tests/runahead/default-future/suite.rc +++ b/tests/runahead/default-future/suite.rc @@ -1,3 +1,4 @@ +#!jinja2 [cylc] UTC mode = True [[event hooks]] @@ -7,17 +8,18 @@ initial cycle point = 20100101T00 final cycle point = 20100105T00 [[dependencies]] - # T00, T01, T02, ... + # foo will fail and cause a stuck bar task at T00. [[[PT1H]]] graph = "foo => bar" - # T04 (depending on T10) - [[[T04/PT6H]]] + # If wibble gets into the pool, it will demand a +PT6H raise + # of the 'runahead limit'. + [[[{{ FUTURE_TRIGGER_START_POINT }}/PT6H]]] graph = """ - baz[+PT6H] => bar + baz[+PT6H] => wibble baz """ [runtime] [[foo]] command scripting = false - [[bar]] + [[bar,baz,wibble]] command scripting = true From f19c8bfb302dbe298b2d4ea4af51aeb33b87879a Mon Sep 17 00:00:00 2001 From: Ben Fitzpatrick Date: Wed, 30 Jul 2014 12:12:46 +0100 Subject: [PATCH 11/14] update fixes and fix backwards comp restart test --- bin/cylc-restart | 15 ++++++++++++++- lib/cylc/cylc_xdot.py | 4 ++-- lib/cylc/graphing.py | 9 +++++---- lib/cylc/taskdef.py | 22 ++++++++++------------ 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/bin/cylc-restart b/bin/cylc-restart index 8eb6f3ca598..5ab506886d5 100755 --- a/bin/cylc-restart +++ b/bin/cylc-restart @@ -31,7 +31,7 @@ from cylc.task_state import task_state from cylc.run import main from cylc.command_prep import prep_file import cylc.TaskID -from cylc.cycling.loader import get_point +from cylc.cycling.loader import get_point, DefaultCycler, ISO8601_CYCLING_TYPE from cylc.wallclock import get_current_time_string @@ -124,6 +124,11 @@ where they got to while the suite was down.""" The state dump file format is: run mode : time :