Add caching support for CWL

src/toil/cwl/cwltoil.py

@@ -4177,8 +4177,12 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
     runtime_context.move_outputs = "leave"
     runtime_context.rm_tmpdir = False
     runtime_context.streaming_allowed = not options.disable_streaming
+    if options.cachedir is not None:
+        runtime_context.cachedir = os.path.abspath(options.cachedir)
     if options.mpi_config_file is not None:
         runtime_context.mpi_config = MpiConfig.load(options.mpi_config_file)
+    if cwltool.main.check_working_directories(runtime_context) is not None:
+        return 1
     setattr(runtime_context, "bypass_file_store", options.bypass_file_store)
     if options.bypass_file_store and options.destBucket:
         # We use the file store to write to buckets, so we can't do this (yet?)
@@ -4360,7 +4364,12 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
             if not options.bypass_file_store:
                 # If we're using the file store we need to start moving output
                 # files now.
-                runtime_context.move_outputs = "move"
+                # But if caching is enabled we have to leave files in the cache directory,
+                # so do a copy if so.
+                if runtime_context.cachedir is not None:
+                    runtime_context.move_outputs = "copy"
+                else:
+                    runtime_context.move_outputs = "move"
 
             # We instantiate an early builder object here to populate indirect
             # secondaryFile references using cwltool's library because we need

src/toil/options/cwl.py

@@ -419,3 +419,12 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         type=str,
         help=suppress_help or "Specify a cloud bucket endpoint for output files.",
     )
+    parser.add_argument(
+        "--cachedir",
+        type=str,
+        help=suppress_help
+        or "Directory to cache intermediate workflow outputs to avoid "
+        "recomputing steps. Can be very helpful in the development and "
+        "troubleshooting of CWL documents.",
+        dest="cachedir"
+    )

src/toil/test/cwl/cwlTest.py

@@ -698,6 +698,30 @@ def path_with_bogus_rev() -> str:
         except subprocess.CalledProcessError:
             pass
 
+    def test_caching(self) -> None:
+        log.info("Running CWL Test Cache.")
+        from toil.cwl import cwltoil
+
+        outDir = self._createTempDir()
+        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
+        cmd = [
+            "--outdir",
+            outDir,
+            "--jobStore",
+            os.path.join(outDir, "jobStore"),
+            "--no-container",
+            "--cachedir",
+            "cache",
+            os.path.join(cwlDir, "revsort.cwl"),
+            os.path.join(cwlDir, "revsort-job.json"),
+        ]
+        # Finish the job with a correct PATH
+        st = StringIO()
+        ret = cwltoil.main(cmd, stdout=st)
+        assert ret == 0
+        # cwltool hashes certain steps into directories, ensure it exists
+        assert os.path.exists(os.path.join(cwlDir, "cache", "9da28e219a61b062824576503f88b863"))
+
     @needs_aws_s3
     def test_streamable(self, extra_args: Optional[list[str]] = None) -> None:
         """