Use a temp path to save local checkpoints for remote save path (#3673)

composer/callbacks/checkpoint_saver.py

@@ -294,10 +294,15 @@ def __init__(
         num_concurrent_uploads: int = 1,
         upload_timeout_in_seconds: int = 3600,
     ):
+
         backend, _, local_folder = parse_uri(str(folder))
         if local_folder == '':
             local_folder = '.'
 
+        is_remote_folder = backend != ''
+        if is_remote_folder:  # If uploading to a remote path, use a temporary directory to save local checkpoints.
+            local_folder = os.path.join(tempfile.mkdtemp(), local_folder)
+
         filename = str(filename)
         remote_file_name = str(remote_file_name) if remote_file_name is not None else None
         latest_filename = str(latest_filename) if latest_filename is not None else None

tests/trainer/test_checkpoint.py

@@ -882,7 +882,8 @@ def _get_tmp_dir(self):
 
                     if delete_local:
                         # delete files locally, forcing trainer to look in object store
-                        shutil.rmtree('first')
+                        assert trainer_1._checkpoint_saver is not None
+                        shutil.rmtree(trainer_1._checkpoint_saver.folder)
 
                     trainer_2 = self.get_trainer(
                         latest_filename=latest_filename,