From 06e81e84b842c9a3ce2436c48debe7c17f70b6ed Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Mon, 2 Sep 2024 11:55:02 +0100 Subject: [PATCH] Multi-slot, multi-channel, & DICOM series --- darwin/dataset/remote_dataset_v2.py | 4 +-- tests/darwin/dataset/remote_dataset_test.py | 29 ++++++++++++++------- tests/darwin/dataset/upload_manager_test.py | 4 +-- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/darwin/dataset/remote_dataset_v2.py b/darwin/dataset/remote_dataset_v2.py index fe62b7000..0c8ce0e97 100644 --- a/darwin/dataset/remote_dataset_v2.py +++ b/darwin/dataset/remote_dataset_v2.py @@ -178,7 +178,7 @@ def push( ---------- files_to_upload : Optional[List[Union[PathLike, LocalFile]]] List of files to upload. These can be folders. - If `item_merge_mode` is set, these must be folders. + If `item_merge_mode` is set, these paths must be folders. blocking : bool, default: True If False, the dataset is not uploaded and a generator function is returned instead. multi_threaded : bool, default: True @@ -259,8 +259,8 @@ def push( files_to_exclude, path, fps, - extract_views, as_frames, + extract_views, preserve_folders, local_files, ) diff --git a/tests/darwin/dataset/remote_dataset_test.py b/tests/darwin/dataset/remote_dataset_test.py index 72dea3d65..4228f7e53 100644 --- a/tests/darwin/dataset/remote_dataset_test.py +++ b/tests/darwin/dataset/remote_dataset_test.py @@ -694,27 +694,35 @@ def test_raises_if_preserve_folders_with_item_merge_mode( def test_find_files_to_upload_merging_slots(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" search_files = [base_path / "jpegs", base_path / "dicoms"] - multi_file_items = _find_files_to_upload_merging(search_files, [], "slots") + local_files, multi_file_items = _find_files_to_upload_merging( + search_files, [], 0, "slots" + ) assert len(multi_file_items) == 2 assert all(isinstance(item, MultiFileItem) for item in multi_file_items) def test_find_files_to_upload_merging_series(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" search_files = [base_path / "dicoms"] - multi_file_items = _find_files_to_upload_merging(search_files, [], "series") + local_files, multi_file_items = _find_files_to_upload_merging( + search_files, [], 0, "series" + ) assert len(multi_file_items) == 1 assert all(isinstance(item, MultiFileItem) for item in multi_file_items) def test_find_files_to_upload_merging_channels(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" search_files = [base_path / "jpegs", base_path / "dicoms"] - multi_file_items = _find_files_to_upload_merging(search_files, [], "channels") + local_files, multi_file_items = _find_files_to_upload_merging( + search_files, [], 0, "channels" + ) assert len(multi_file_items) == 2 def test_find_files_to_upload_merging_does_not_search_recursively(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir2" search_files = [base_path / "recursive_search"] - multi_file_items = _find_files_to_upload_merging(search_files, [], "slots") + local_files, multi_file_items = _find_files_to_upload_merging( + search_files, [], 0, "slots" + ) assert len(multi_file_items) == 1 assert len(multi_file_items[0].files) == 2 @@ -765,31 +773,32 @@ def setup_zip(self): def test_create_multi_file_item_slots(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" / "jpegs" files = natsorted(list(base_path.glob("*"))) - item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SLOTS) + item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SLOTS, fps=0) assert len(item.files) == 6 assert item.name == "jpegs" assert item.layout == { "version": 2, "slots": ["0", "1", "2", "3", "4", "5"], "type": "grid", + "layout_shape": [3, 2], } def test_create_multi_file_item_series(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" / "dicoms" files = natsorted(list(base_path.glob("*"))) - item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES) + item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES, fps=0) assert len(item.files) == 6 assert item.name == "dicoms" assert item.layout == { "version": 2, - "slots": ["0", "1", "2", "3", "4", "5"], + "slots": ["dicoms"], "type": "grid", } def test_create_multi_file_item_channels(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir1" / "jpegs" files = natsorted(list(base_path.glob("*"))) - item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS) + item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS, fps=0) assert len(item.files) == 6 assert item.name == "jpegs" assert item.layout == { @@ -803,7 +812,7 @@ def test_create_series_no_valid_files(self, setup_zip): with pytest.raises( ValueError, match="No `.dcm` files found in 1st level of directory" ): - MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES) + MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES, fps=0) def test_create_channels_too_many_files(self, setup_zip): base_path = setup_zip / "push_test_dir" / "dir2" / "too_many_channels" @@ -812,7 +821,7 @@ def test_create_channels_too_many_files(self, setup_zip): ValueError, match=r"No multi-channel item can have more than 16 files. The following directory has 17 files: .*", ): - MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS) + MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS, fps=0) @pytest.mark.usefixtures("file_read_write_test") diff --git a/tests/darwin/dataset/upload_manager_test.py b/tests/darwin/dataset/upload_manager_test.py index 03e65be37..3ae764d16 100644 --- a/tests/darwin/dataset/upload_manager_test.py +++ b/tests/darwin/dataset/upload_manager_test.py @@ -107,7 +107,7 @@ def test_pending_count_is_correct(dataset: RemoteDataset, request_upload_endpoin assert pending_item.dataset_item_id == "3b241101-e2bb-4255-8caf-4136c566a964" assert pending_item.filename == "test.jpg" assert pending_item.path == "/" - assert pending_item.reason is None + assert pending_item.reason[0] is None @pytest.mark.usefixtures("file_read_write_test") @@ -149,7 +149,7 @@ def test_blocked_count_is_correct(dataset: RemoteDataset, request_upload_endpoin assert blocked_item.dataset_item_id == "3b241101-e2bb-4255-8caf-4136c566a964" assert blocked_item.filename == "test.jpg" assert blocked_item.path == "/" - assert blocked_item.reason == "ALREADY_EXISTS" + assert blocked_item.reason[0] == "ALREADY_EXISTS" @pytest.mark.usefixtures("file_read_write_test")