Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix operation group status #774

Merged
merged 9 commits into from
Oct 19, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions changelog.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,17 @@ Changelog
The **signac-flow** package follows `semantic versioning <https://semver.org/>`_.
The numbers in brackets denote the related GitHub issue and/or pull request.

Version 0.27
============

[0.27.0] -- 202x-xx-xx
----------------------

Fixed
+++++

- Restored static output of non-singleton groups (#773, #774).

Version 0.26
============

Expand Down
77 changes: 50 additions & 27 deletions flow/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -2721,7 +2721,7 @@
)
return result

status_groups = set(self._gather_flow_groups(names))
status_groups = self._gather_selected_flow_groups(names)

with self._buffered():
aggregate_groups = list(
Expand Down Expand Up @@ -3692,7 +3692,7 @@
# Generate _JobOperation instances for selected groups and aggregates.
with self._buffered():
operations = []
run_groups = set(self._gather_flow_groups(names))
run_groups = set(self._gather_executable_flow_groups(names))
for (
aggregate_id,
aggregate,
Expand Down Expand Up @@ -3754,17 +3754,16 @@
operations, pretend=pretend, np=np, timeout=timeout, progress=progress
)

def _gather_flow_groups(self, names=None):
def _gather_selected_flow_groups(self, names=None):
r"""Grabs :class:`~.FlowGroup`\ s that match any of a set of names.

The provided names can be any regular expressions that fully match a group name.
The provided names can be any regular expression that fully match a group name.
b-butler marked this conversation as resolved.
Show resolved Hide resolved

Parameters
----------
names : iterable of :class:`str`
Only select operations that match the provided set of names
(interpreted as regular expressions), or all if the argument is
None. (Default value = None)
Only select groups that match the provided set of names (interpreted as regular
expressions), or all if the argument is None. (Default value = None)

Returns
-------
Expand All @@ -3773,24 +3772,48 @@

"""
if names is None:
# If no names are selected, use all singleton groups
operations = [self._groups[name] for name in self.operations]
else:
operations = {}
for name in names:
if name in operations:
continue
groups = [
group
for group_name, group in self.groups.items()
if re.fullmatch(name, group_name)
]
if len(groups) > 0:
for group in groups:
operations[group.name] = group
else:
continue
operations = list(operations.values())
return list(self._groups.values())
operations = {}
for name in names:
if name in operations:
continue
groups = [
group
for group_name, group in self.groups.items()
if re.fullmatch(name, group_name)
]
for group in groups:
operations[group.name] = group
return list(operations.values())

def _gather_executable_flow_groups(self, names=None):
r"""Grabs immediately executable flow groups that match any given name.

The provided names can be any regular expression that fully match a group name.

Note
----
The behavior is distinct from ``_gather_selected_flow_groups`` in that
for execution the default set is not all groups but all singelton
groups (operations).
b-butler marked this conversation as resolved.
Show resolved Hide resolved

Parameters
----------
names : iterable of :class:`str`
Only select groups that match the provided set of names (interpreted as regular
expressions), or all singleton groups if the argument is None. (Default value = None)

Returns
-------
list
List of groups matching the provided names.

"""
if names is None:
return [self._groups[op_name] for op_name in self.operations]
operations = self._gather_selected_flow_groups(names)
# Have to verify no overlap to ensure all returned groups are
# simultaneously executable.
if not FlowProject._verify_group_compatibility(operations):
raise ValueError(
"Cannot specify groups or operations that will be included "
Expand Down Expand Up @@ -3839,7 +3862,7 @@
been collected appropriately from its contained operations.

"""
submission_groups = set(self._gather_flow_groups(names))
submission_groups = set(self._gather_executable_flow_groups(names))

# Fetch scheduler status
scheduler_info = self._query_scheduler_status()
Expand Down Expand Up @@ -4494,7 +4517,7 @@
if operation_names is None:
selected_groups = {self._groups[name] for name in self.operations}
else:
selected_groups = set(self._gather_flow_groups(operation_names))
selected_groups = set(self._gather_executable_flow_groups(operation_names))

Check warning on line 4520 in flow/project.py

View check run for this annotation

Codecov / codecov/patch

flow/project.py#L4520

Added line #L4520 was not covered by tests
for (
aggregate_id,
aggregate,
Expand Down
Binary file modified tests/status_reference_data.tar.gz
Binary file not shown.
4 changes: 2 additions & 2 deletions tests/test_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ def test_groups(self, groups, get_status):
assert group in operations_output

def test_operation_in_group(self, get_status):
with pytest.raises(ValueError):
get_status(operation=["op1", "group1"])
# Ensure groups with overlapping operations work
get_status(operation=["op1", "group1"])


class TestProjectStatusNoEligibleOperations(TestProjectBase):
Expand Down