nayduck: support reading commented ./<path> include directives

When check_pytest.py and check_nightly.py read the test list files
handle commented out include directives (i.e. `#./<path>` lines) so
that the include can be commented out with a TODO and the check
scripts will treat files listed in those files as being mentioned.

Issue: near#4362
Issue: near#4552

nightly/TODO-disabled-4552.txt

@@ -1,14 +1,8 @@
-# TODO(#4552): Those tests were identified as missing from any of the test list
-# files.  It’s not entirely clear why so some investigation is necessary to
-# figure that out.  The tests need to either be enabled or removed completely if
-# they are no longer needed/valid.
-# pytest adversarial/gc_rollback.py
-# pytest adversarial/gc_rollback.py --features nightly_protocol,nightly_protocol_features
-# pytest sanity/restart.py
-# pytest sanity/restart.py --features nightly_protocol,nightly_protocol_features
-# pytest sanity/rpc_finality.py
-# pytest sanity/rpc_finality.py --features nightly_protocol,nightly_protocol_features
-# pytest sanity/state_migration.py
-# pytest sanity/state_migration.py --features nightly_protocol,nightly_protocol_features
-# pytest stress/network_stress.py
-# pytest stress/network_stress.py --features nightly_protocol,nightly_protocol_features
+pytest adversarial/gc_rollback.py
+pytest adversarial/gc_rollback.py --features nightly_protocol,nightly_protocol_features
+pytest sanity/restart.py
+pytest sanity/restart.py --features nightly_protocol,nightly_protocol_features
+pytest sanity/rpc_finality.py
+pytest sanity/rpc_finality.py --features nightly_protocol,nightly_protocol_features
+pytest sanity/state_migration.py
+pytest sanity/state_migration.py --features nightly_protocol,nightly_protocol_features

nightly/nightly.txt

@@ -1,4 +1,8 @@
-./TODO-disabled-4552.txt
+# TODO(#4552): Those tests were identified as missing from any of the test list
+# files.  It’s not clear why so some investigation is necessary.  The tests need
+# to either be enabled or removed completely if they are no longer needed/valid.
+#./TODO-disabled-4552.txt
+
 ./sandbox.txt
 ./pytest.txt
 ./expensive.txt

nightly/pytest-stress.txt

@@ -19,5 +19,6 @@ pytest --timeout=2000 stress/stress.py 3 3 3 0 staking transactions node_restart
 pytest --timeout=300 stress/saturate_routing_table.py
 pytest --timeout=300 stress/saturate_routing_table.py --features nightly_protocol,nightly_protocol_features
 
+# TODO(#4552): Enable this at some point.
 # pytest stress/network_stress.py
 # pytest stress/network_stress.py --features nightly_protocol,nightly_protocol_features

scripts/nayduck.py

@@ -67,6 +67,12 @@ def read_tests_from_file(path: pathlib.Path, *,
     as if they were directly in the source file.  The `./<path>` directives are
     handled recursively up to three levels deep.
 
+    If include_comments is True, `#./<path>` lines are handled as well with all
+    included line commented out.  This is useful to comment out a include a with
+    TODO comment included and have check_nightly.py and check_pytest.py scripts
+    still recognise those includes.  Note that the line must start with `#./`,
+    i.e. there must be no space between hash and the dot.
+
     Args:
         path: Path to the file to read.
         include_comments: By default empty lines and lines whose first non-space
@@ -78,16 +84,23 @@ def read_tests_from_file(path: pathlib.Path, *,
         An iterable over lines in the given file.  All lines are stripped of
         leading and trailing white space.
     """
-    def impl(path: pathlib.Path, depth: int):
+
+    def impl(path: pathlib.Path, depth: int, comment: bool = False):
         for lineno, line in enumerate(reader(path).splitlines()):
-            line = line.strip()
-            if line.startswith('./'):
+            line = line.rstrip()
+            if line.startswith('./') or (include_comments and
+                                         line.startswith('#./')):
                 if depth == 3:
                     print(f'{path}:{lineno+1}: ignoring {line}; '
                           f'would exceed depth limit of {depth}')
                 else:
-                    yield from impl(path.parent / line, depth + 1)
+                    incpath = line[1:] if line.startswith('#') else line
+                    yield from impl(path.parent / incpath,
+                                    depth + 1,
+                                    comment or line.startswith('#'))
             elif include_comments or (line and line[0] != '#'):
+                if comment and not line.startswith('#'):
+                    line = '#' + line
                 yield line
 
     return impl(path, 1)