Merge pull request #44 from commit-0/rebuild

Rebuild

.github/workflows/system.yml

@@ -25,9 +25,19 @@ jobs:
       - name: Get tests
         run: uv run commit0 get-tests simpy
       - name: Test
-        run: uv run commit0 test simpy tests/test_event.py::test_succeed --reference
+        env:
+          MODAL_TOKEN_ID: ${{secrets.MODAL_TOKEN_ID}}
+          MODAL_TOKEN_SECRET: ${{secrets.MODAL_TOKEN_SECRET}}
+        run: |
+          uv run commit0 test simpy tests/test_event.py::test_succeed --reference --rebuild
+          uv run commit0 test simpy tests/test_event.py::test_succeed --reference
       - name: Evaluate
-        run: uv run commit0 evaluate --reference
+        env:
+          MODAL_TOKEN_ID: ${{secrets.MODAL_TOKEN_ID}}
+          MODAL_TOKEN_SECRET: ${{secrets.MODAL_TOKEN_SECRET}}
+        run: |
+          uv run commit0 evaluate --reference  --rebuild
+          uv run commit0 evaluate --reference
       - name: Lint
         run: uv run commit0 lint commit0/harness/lint.py
       - name: Save

commit0/cli.py

@@ -222,6 +222,9 @@ def test(
     reference: Annotated[
         bool, typer.Option("--reference", help="Test the reference commit.")
     ] = False,
+    rebuild: bool = typer.Option(
+        False, "--rebuild", help="Whether to rebuild an image"
+    ),
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
         help="Path to the commit0 dot file, where the setup config is stored",
@@ -266,6 +269,7 @@ def test(
         backend,
         timeout,
         num_cpus,
+        rebuild,
         verbose,
     )
 
@@ -286,6 +290,7 @@ def evaluate(
         ".commit0.yaml",
         help="Path to the commit0 dot file, where the setup config is stored",
     ),
+    rebuild: bool = typer.Option(False, "--rebuild", help="Whether to rebuild images"),
 ) -> None:
     """Evaluate Commit0 split you choose in Setup Stage."""
     check_commit0_path()
@@ -314,6 +319,7 @@ def evaluate(
         timeout,
         num_cpus,
         num_workers,
+        rebuild,
     )
 
 

commit0/harness/evaluate.py

@@ -28,6 +28,7 @@ def main(
     timeout: int,
     num_cpus: int,
     num_workers: int,
+    rebuild_image: bool,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     repos = SPLIT[repo_split]
@@ -57,6 +58,7 @@ def main(
                     backend,
                     timeout,
                     num_cpus,
+                    rebuild_image=rebuild_image,
                     verbose=0,
                 ): None
                 for repo, test_dir in pairs

commit0/harness/execution_context.py

@@ -44,6 +44,7 @@ def __init__(
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
         files_to_collect: Optional[list[str]] = None,
+        rebuild_image: bool = False,
     ):
         """Create the remote execution context
 
@@ -85,6 +86,7 @@ def __init__(
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
         files_to_collect: Optional[list[str]] = None,
+        rebuild_image: bool = False,
     ):
         super().__init__(
             spec,
@@ -145,6 +147,7 @@ def __init__(
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
         files_to_collect: Optional[list[str]] = None,
+        rebuild_image: bool = False,
     ):
         super().__init__(
             spec,
@@ -161,7 +164,7 @@ def __init__(
         # the image must exist on dockerhub
         reponame = spec.repo.split("/")[-1]
         image_name = f"wentingzhao/{reponame}:latest".lower()
-        image = modal.Image.from_registry(image_name)
+        image = modal.Image.from_registry(image_name, force_build=rebuild_image)
         if files_to_copy:
             for _, f in files_to_copy.items():
                 image = image.copy_local_file(f["src"], f["dest"])  # type: ignore
@@ -175,9 +178,8 @@ def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
                 command += " && "
                 for fname in self.files_to_collect:
                     remote_file = Path(self.spec.repo_directory) / fname
-                    cp_cmd = f"cp {str(remote_file)} /vol/{fname} 2>/dev/null; "
+                    cp_cmd = f"test -e {str(remote_file)} && cp {str(remote_file)} /vol/{fname}; "
                     command += cp_cmd
-
             self.sandbox = modal.Sandbox.create(
                 "bash",
                 "-c",
@@ -198,7 +200,9 @@ def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
                 timed_out = False
 
             if self.files_to_collect:
-                for fname in self.files_to_collect:
+                fnames = vol.listdir("")
+                for fname in fnames:
+                    fname = fname.path
                     with (self.log_dir / fname).open("wb") as f:
                         for data in vol.read_file(fname):
                             f.write(data)

commit0/harness/run_pytest_ids.py

@@ -37,6 +37,7 @@ def main(
     backend: str,
     timeout: int,
     num_cpus: int,
+    rebuild_image: bool,
     verbose: int,
 ) -> None:
     """Runs the pytests for repos in a dataset.
@@ -125,7 +126,14 @@ def main(
 
     try:
         with execution_context(
-            spec, logger, timeout, num_cpus, log_dir, files_to_copy, files_to_collect
+            spec,
+            logger,
+            timeout,
+            num_cpus,
+            log_dir,
+            files_to_copy,
+            files_to_collect,
+            rebuild_image,
         ) as context:
             output, timed_out, total_runtime = context.exec_run_with_timeout(
                 "/bin/bash /eval.sh"