Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix batch delete with operands #479

Merged
merged 5 commits into from
Sep 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 94 additions & 34 deletions integration/test_batch.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import uuid
from dataclasses import dataclass
from typing import Union, Sequence, Optional
from typing import List, Union, Sequence, Optional

import pytest

Expand Down Expand Up @@ -58,6 +58,7 @@ def client():
"properties": [
{"name": "test", "dataType": ["Test"]},
{"name": "name", "dataType": ["string"]},
{"name": "names", "dataType": ["string[]"]},
],
"vectorizer": "none",
}
Expand All @@ -83,55 +84,114 @@ def test_add_data_object(client: weaviate.Client, uuid: Optional[UUID], vector:
assert has_batch_errors(response) is False, str(response)


def test_delete_objects(client: weaviate.Client):
with client.batch as batch:
batch.add_data_object(data_object={"name": "one"}, class_name="Test")
batch.add_data_object(data_object={"name": "two"}, class_name="test")
batch.add_data_object(data_object={"name": "three"}, class_name="Test")
batch.add_data_object(data_object={"name": "four"}, class_name="test")
batch.add_data_object(data_object={"name": "five"}, class_name="Test")

with client.batch as batch:
batch.delete_objects(
"Test",
where={
@pytest.mark.parametrize(
"objs,where",
[
(
[
{"name": "zero"},
],
{
"path": ["name"],
"operator": "NotEqual",
"valueText": "one",
},
),
(
[
{"name": "one"},
],
{
"path": ["name"],
"operator": "Equal",
"valueText": "one",
},
)
res = client.data_object.get()
names = [obj["properties"]["name"] for obj in res["objects"]]
assert "one" not in names

with client.batch as batch:
batch.delete_objects(
"test",
where={
),
(
[{"name": "two"}, {"name": "three"}],
{
"path": ["name"],
"operator": "ContainsAny",
"valueTextArray": ["two", "three"],
},
)
res = client.data_object.get()
names = [obj["properties"]["name"] for obj in res["objects"]]
assert "two" not in names
assert "three" not in names
),
(
[
{"names": ["Tim", "Tom"], "name": "four"},
],
{
"path": ["names"],
"operator": "ContainsAll",
"valueTextArray": ["Tim", "Tom"],
},
),
(
[
{"names": ["Tim", "Tom"], "name": "five"},
],
{
"operator": "And",
"operands": [
{
"path": ["names"],
"operator": "ContainsAll",
"valueTextArray": ["Tim", "Tom"],
},
{
"path": ["name"],
"operator": "Equal",
"valueText": "five",
},
],
},
),
(
[{"name": "six"}, {"name": "seven"}],
{
"operator": "Or",
"operands": [
{
"path": ["name"],
"operator": "Equal",
"valueText": "six",
},
{
"path": ["name"],
"operator": "Equal",
"valueText": "seven",
},
],
},
),
(
[
{"name": "eight"},
],
{
"path": ["name"],
"operator": "Like",
"valueText": "eig*",
},
),
],
)
def test_delete_objects_successes(client: weaviate.Client, objs: List[dict], where: dict):
with client.batch as batch:
for obj in objs:
batch.add_data_object(data_object=obj, class_name="Test")

with client.batch as batch:
batch.delete_objects(
"Test",
where={
"path": ["name"],
"operator": "ContainsAll",
"valueTextArray": ["four", "five"],
},
where=where,
)
res = client.data_object.get()
names = [obj["properties"]["name"] for obj in res["objects"]]
assert "four" in names
assert "five" in names
for obj in objs:
assert obj.get("name") not in names


def test_delete_objects_errors(client: weaviate.Client):
with pytest.raises(ValueError) as error:
with client.batch as batch:
batch.delete_objects(
Expand Down
35 changes: 22 additions & 13 deletions weaviate/batch/crud_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -1812,21 +1812,30 @@ def _clean_delete_objects_where(where: dict) -> dict:
dict
The Weaviate-defined where filter.
"""
py_value_type = _find_value_type(where)
weaviate_value_type = _convert_value_type(py_value_type)
if "operator" not in where:
raise ValueError("Where filter is missing required field `operator`." f" Given: {where}")
if where["operator"] not in WHERE_OPERATORS:
raise ValueError(
f"Operator {where['operator']} is not allowed. "
f"Allowed operators are: {WHERE_OPERATORS}"
)
operator = where["operator"]
if "Contains" in operator and "Array" not in weaviate_value_type:
if "path" in where:
py_value_type = _find_value_type(where)
weaviate_value_type = _convert_value_type(py_value_type)
if "operator" not in where:
raise ValueError(
"Where filter is missing required field `operator`." f" Given: {where}"
)
if where["operator"] not in WHERE_OPERATORS:
raise ValueError(
f"Operator {where['operator']} is not allowed. "
f"Allowed operators are: {WHERE_OPERATORS}"
)
operator = where["operator"]
if "Contains" in operator and "Array" not in weaviate_value_type:
raise ValueError(
f"Operator '{operator}' is not supported for value type '{weaviate_value_type}'. Supported value types are: {VALUE_ARRAY_TYPES}"
)
where[weaviate_value_type] = where.pop(py_value_type)
elif "operands" in where:
where["operands"] = [_clean_delete_objects_where(operand) for operand in where["operands"]]
else:
raise ValueError(
f"Operator '{operator}' is not supported for value type '{weaviate_value_type}'. Supported value types are: {VALUE_ARRAY_TYPES}"
"Where is missing required fields `path` or `operands`." f" Given: {where}"
)
where[weaviate_value_type] = where.pop(py_value_type)
return where


Expand Down