Skip to content

Commit

Permalink
Multi-slot, multi-channel, & DICOM series
Browse files Browse the repository at this point in the history
  • Loading branch information
@JBWilkie
JBWilkie committed Sep 2, 2024
1 parent e066974 commit 06e81e8
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 14 deletions.
4 changes: 2 additions & 2 deletions darwin/dataset/remote_dataset_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def push(
----------
files_to_upload : Optional[List[Union[PathLike, LocalFile]]]
List of files to upload. These can be folders.
If `item_merge_mode` is set, these must be folders.
If `item_merge_mode` is set, these paths must be folders.
blocking : bool, default: True
If False, the dataset is not uploaded and a generator function is returned instead.
multi_threaded : bool, default: True
Expand Down Expand Up @@ -259,8 +259,8 @@ def push(
files_to_exclude,
path,
fps,
extract_views,
as_frames,
extract_views,
preserve_folders,
local_files,
)
Expand Down
29 changes: 19 additions & 10 deletions tests/darwin/dataset/remote_dataset_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -694,27 +694,35 @@ def test_raises_if_preserve_folders_with_item_merge_mode(
def test_find_files_to_upload_merging_slots(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1"
search_files = [base_path / "jpegs", base_path / "dicoms"]
multi_file_items = _find_files_to_upload_merging(search_files, [], "slots")
local_files, multi_file_items = _find_files_to_upload_merging(
search_files, [], 0, "slots"
)
assert len(multi_file_items) == 2
assert all(isinstance(item, MultiFileItem) for item in multi_file_items)

def test_find_files_to_upload_merging_series(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1"
search_files = [base_path / "dicoms"]
multi_file_items = _find_files_to_upload_merging(search_files, [], "series")
local_files, multi_file_items = _find_files_to_upload_merging(
search_files, [], 0, "series"
)
assert len(multi_file_items) == 1
assert all(isinstance(item, MultiFileItem) for item in multi_file_items)

def test_find_files_to_upload_merging_channels(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1"
search_files = [base_path / "jpegs", base_path / "dicoms"]
multi_file_items = _find_files_to_upload_merging(search_files, [], "channels")
local_files, multi_file_items = _find_files_to_upload_merging(
search_files, [], 0, "channels"
)
assert len(multi_file_items) == 2

def test_find_files_to_upload_merging_does_not_search_recursively(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir2"
search_files = [base_path / "recursive_search"]
multi_file_items = _find_files_to_upload_merging(search_files, [], "slots")
local_files, multi_file_items = _find_files_to_upload_merging(
search_files, [], 0, "slots"
)
assert len(multi_file_items) == 1
assert len(multi_file_items[0].files) == 2

Expand Down Expand Up @@ -765,31 +773,32 @@ def setup_zip(self):
def test_create_multi_file_item_slots(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1" / "jpegs"
files = natsorted(list(base_path.glob("*")))
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SLOTS)
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SLOTS, fps=0)
assert len(item.files) == 6
assert item.name == "jpegs"
assert item.layout == {
"version": 2,
"slots": ["0", "1", "2", "3", "4", "5"],
"type": "grid",
"layout_shape": [3, 2],
}

def test_create_multi_file_item_series(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1" / "dicoms"
files = natsorted(list(base_path.glob("*")))
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES)
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES, fps=0)
assert len(item.files) == 6
assert item.name == "dicoms"
assert item.layout == {
"version": 2,
"slots": ["0", "1", "2", "3", "4", "5"],
"slots": ["dicoms"],
"type": "grid",
}

def test_create_multi_file_item_channels(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir1" / "jpegs"
files = natsorted(list(base_path.glob("*")))
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS)
item = MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS, fps=0)
assert len(item.files) == 6
assert item.name == "jpegs"
assert item.layout == {
Expand All @@ -803,7 +812,7 @@ def test_create_series_no_valid_files(self, setup_zip):
with pytest.raises(
ValueError, match="No `.dcm` files found in 1st level of directory"
):
MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES)
MultiFileItem(base_path, files, merge_mode=ItemMergeMode.SERIES, fps=0)

def test_create_channels_too_many_files(self, setup_zip):
base_path = setup_zip / "push_test_dir" / "dir2" / "too_many_channels"
Expand All @@ -812,7 +821,7 @@ def test_create_channels_too_many_files(self, setup_zip):
ValueError,
match=r"No multi-channel item can have more than 16 files. The following directory has 17 files: .*",
):
MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS)
MultiFileItem(base_path, files, merge_mode=ItemMergeMode.CHANNELS, fps=0)


@pytest.mark.usefixtures("file_read_write_test")
Expand Down
4 changes: 2 additions & 2 deletions tests/darwin/dataset/upload_manager_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def test_pending_count_is_correct(dataset: RemoteDataset, request_upload_endpoin
assert pending_item.dataset_item_id == "3b241101-e2bb-4255-8caf-4136c566a964"
assert pending_item.filename == "test.jpg"
assert pending_item.path == "/"
assert pending_item.reason is None
assert pending_item.reason[0] is None


@pytest.mark.usefixtures("file_read_write_test")
Expand Down Expand Up @@ -149,7 +149,7 @@ def test_blocked_count_is_correct(dataset: RemoteDataset, request_upload_endpoin
assert blocked_item.dataset_item_id == "3b241101-e2bb-4255-8caf-4136c566a964"
assert blocked_item.filename == "test.jpg"
assert blocked_item.path == "/"
assert blocked_item.reason == "ALREADY_EXISTS"
assert blocked_item.reason[0] == "ALREADY_EXISTS"


@pytest.mark.usefixtures("file_read_write_test")
Expand Down

0 comments on commit 06e81e8

Please sign in to comment.