From 4ce433ee6a3ac8ef995b6d683a2a2a2b657bfcef Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 4 Apr 2024 12:50:23 +0100 Subject: [PATCH 1/3] Re-added V1 filter compatibility + some formatting --- darwin/dataset/remote_dataset_v2.py | 5 +++++ darwin/importer/importer.py | 26 ++++++++++++++------------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/darwin/dataset/remote_dataset_v2.py b/darwin/dataset/remote_dataset_v2.py index eabef6c2c..bfa394630 100644 --- a/darwin/dataset/remote_dataset_v2.py +++ b/darwin/dataset/remote_dataset_v2.py @@ -284,6 +284,11 @@ def fetch_remote_files( post_sort: Dict[str, str] = {} if filters: + # Backward compatibility with V1 filter parameter + if "filenames" in filters: + filters["item_names"] = filters["filenames"] + del filters["filenames"] + for list_type in [ "item_names", "statuses", diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index d6966a8d7..55a3a6d3c 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -282,9 +282,9 @@ def _get_team_properties_annotation_lookup(client): team_properties = client.get_team_properties() # (property-name, annotation_class_id): FullProperty object - team_properties_annotation_lookup: Dict[Tuple[str, Optional[int]], FullProperty] = ( - {} - ) + team_properties_annotation_lookup: Dict[ + Tuple[str, Optional[int]], FullProperty + ] = {} for prop in team_properties: team_properties_annotation_lookup[(prop.name, prop.annotation_class_id)] = prop @@ -395,9 +395,11 @@ def _import_properties( # raise error if annotation-property not present in metadata if (annotation_name, a_prop.name) not in metadata_cls_prop_lookup: - # check if they are present in team properties - if (a_prop.name, annotation_class_id) in team_properties_annotation_lookup: + if ( + a_prop.name, + annotation_class_id, + ) in team_properties_annotation_lookup: # get team property t_prop: FullProperty = team_properties_annotation_lookup[ (a_prop.name, annotation_class_id) @@ -411,7 +413,7 @@ def _import_properties( ] = set() continue - # get team property value + # get team property value t_prop_val = None for prop_val in t_prop.property_values or []: if prop_val.value == a_prop.value: @@ -921,9 +923,9 @@ def import_annotations( # noqa: C901 # Need to re parse the files since we didn't save the annotations in memory for local_path in set(local_file.path for local_file in local_files): # noqa: C401 - imported_files: Union[List[dt.AnnotationFile], dt.AnnotationFile, None] = ( - importer(local_path) - ) + imported_files: Union[ + List[dt.AnnotationFile], dt.AnnotationFile, None + ] = importer(local_path) if imported_files is None: parsed_files = [] elif not isinstance(imported_files, List): @@ -1323,9 +1325,9 @@ def _import_annotations( # Insert the default slot name if not available in the import source annotation = _handle_slot_names(annotation, dataset.version, default_slot_name) - annotation_class_ids_map[(annotation_class.name, annotation_type)] = ( - annotation_class_id - ) + annotation_class_ids_map[ + (annotation_class.name, annotation_type) + ] = annotation_class_id serial_obj = { "annotation_class_id": annotation_class_id, "data": data, From 5d5bfed93425be23eb8b8ce433060103609c6306 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 4 Apr 2024 12:52:52 +0100 Subject: [PATCH 2/3] Corrected invocations of fetch_remote_files with V2 parameter --- darwin/cli_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/darwin/cli_functions.py b/darwin/cli_functions.py index 662280e75..f814217dd 100644 --- a/darwin/cli_functions.py +++ b/darwin/cli_functions.py @@ -1026,7 +1026,7 @@ def set_file_status(dataset_slug: str, status: str, files: List[str]) -> None: dataset_identifier=dataset_slug ) items: Iterator[DatasetItem] = dataset.fetch_remote_files( - {"filenames": ",".join(files)} + {"item_names": ",".join(files)} ) if status == "archived": dataset.archive(items) @@ -1066,7 +1066,7 @@ def delete_files( dataset: RemoteDataset = client.get_remote_dataset( dataset_identifier=dataset_slug ) - items, items_2 = tee(dataset.fetch_remote_files({"filenames": files})) + items, items_2 = tee(dataset.fetch_remote_files({"item_names": files})) if not skip_user_confirmation and not secure_continue_request(): console.print("Cancelled.") return @@ -1281,7 +1281,7 @@ def post_comment( _error(f"unable to find dataset: {dataset_slug}") items: List[DatasetItem] = list( - dataset.fetch_remote_files(filters={"filenames": [filename]}) + dataset.fetch_remote_files(filters={"item_names": [filename]}) ) if len(items) == 0: From fd7fbd6aeefd9dd68fc16864b726a1b3658fa001 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 4 Apr 2024 13:03:26 +0100 Subject: [PATCH 3/3] Fixed failing tests --- tests/darwin/cli_functions_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/darwin/cli_functions_test.py b/tests/darwin/cli_functions_test.py index 718c92433..7ddf1a78f 100644 --- a/tests/darwin/cli_functions_test.py +++ b/tests/darwin/cli_functions_test.py @@ -272,7 +272,7 @@ def test_calls_dataset_archive( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": "one.jpg,two.jpg"} + {"item_names": "one.jpg,two.jpg"} ) mock.assert_called_once_with(fetch_remote_files_mock.return_value) @@ -291,7 +291,7 @@ def test_calls_dataset_clear( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": "one.jpg,two.jpg"} + {"item_names": "one.jpg,two.jpg"} ) mock.assert_called_once_with(fetch_remote_files_mock.return_value) @@ -310,7 +310,7 @@ def test_calls_dataset_new( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": "one.jpg,two.jpg"} + {"item_names": "one.jpg,two.jpg"} ) mock.assert_called_once_with(fetch_remote_files_mock.return_value) @@ -331,7 +331,7 @@ def test_calls_dataset_restore_archived( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": "one.jpg,two.jpg"} + {"item_names": "one.jpg,two.jpg"} ) mock.assert_called_once_with(fetch_remote_files_mock.return_value) @@ -356,7 +356,7 @@ def test_bypasses_user_prompt_if_yes_flag_is_true( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": ["one.jpg", "two.jpg"]} + {"item_names": ["one.jpg", "two.jpg"]} ) mock.assert_called_once() @@ -376,7 +376,7 @@ def test_deletes_items_if_user_accepts_prompt( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": ["one.jpg", "two.jpg"]} + {"item_names": ["one.jpg", "two.jpg"]} ) mock.assert_called_once() @@ -396,7 +396,7 @@ def test_does_not_delete_items_if_user_refuses_prompt( dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": ["one.jpg", "two.jpg"]} + {"item_names": ["one.jpg", "two.jpg"]} ) mock.assert_not_called() @@ -422,7 +422,7 @@ def error_mock(): dataset_identifier=dataset_identifier ) fetch_remote_files_mock.assert_called_once_with( - {"filenames": ["one.jpg", "two.jpg"]} + {"item_names": ["one.jpg", "two.jpg"]} ) mock.assert_called_once() exception.assert_called_once_with(1)