From 10b60adf53c6db943b663b786efb55bebf8503ec Mon Sep 17 00:00:00 2001 From: Onur Ozer Date: Mon, 10 Jun 2024 13:05:11 -0700 Subject: [PATCH 1/6] feat(datastore): Support "IN" query filter --- datastore/gcloud/aio/datastore/__init__.py | 2 + datastore/gcloud/aio/datastore/constants.py | 1 + datastore/gcloud/aio/datastore/filter.py | 15 ++-- datastore/tests/integration/smoke_test.py | 76 +++++++++++++++++++++ 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/datastore/gcloud/aio/datastore/__init__.py b/datastore/gcloud/aio/datastore/__init__.py index 14cf87ba6..fe7e558ae 100644 --- a/datastore/gcloud/aio/datastore/__init__.py +++ b/datastore/gcloud/aio/datastore/__init__.py @@ -181,6 +181,7 @@ class MyGQLQuery(gcloud.aio.datastore.GQLQuery): """ import importlib.metadata +from .array import Array from .constants import CompositeFilterOperator from .constants import Consistency from .constants import Direction @@ -215,6 +216,7 @@ class MyGQLQuery(gcloud.aio.datastore.GQLQuery): __version__ = importlib.metadata.version('gcloud-aio-datastore') __all__ = [ + 'Array', 'CompositeFilter', 'CompositeFilterOperator', 'Consistency', diff --git a/datastore/gcloud/aio/datastore/constants.py b/datastore/gcloud/aio/datastore/constants.py index 1b2aeb4a9..ab2497d20 100644 --- a/datastore/gcloud/aio/datastore/constants.py +++ b/datastore/gcloud/aio/datastore/constants.py @@ -53,6 +53,7 @@ class PropertyFilterOperator(enum.Enum): LESS_THAN_OR_EQUAL = 'LESS_THAN_OR_EQUAL' NOT_EQUAL = 'NOT_EQUAL' UNSPECIFIED = 'OPERATOR_UNSPECIFIED' + IN = 'IN' class ResultType(enum.Enum): diff --git a/datastore/gcloud/aio/datastore/filter.py b/datastore/gcloud/aio/datastore/filter.py index ec66dc1ea..c9389460b 100644 --- a/datastore/gcloud/aio/datastore/filter.py +++ b/datastore/gcloud/aio/datastore/filter.py @@ -1,7 +1,9 @@ from typing import Any from typing import Dict from typing import List +from typing import Union +from .array import Array from .constants import CompositeFilterOperator from .constants import PropertyFilterOperator from .value import Value @@ -89,7 +91,7 @@ class PropertyFilter(BaseFilter): def __init__( self, prop: str, operator: PropertyFilterOperator, - value: Value, + value: Union[Value, Array] ) -> None: self.prop = prop self.operator = operator @@ -113,8 +115,13 @@ def from_repr(cls, data: Dict[str, Any]) -> 'PropertyFilter': return cls(prop=prop, operator=operator, value=value) def to_repr(self) -> Dict[str, Any]: - return { + rep = { 'op': self.operator.value, - 'property': {'name': self.prop}, - 'value': self.value.to_repr(), + 'property': {'name': self.prop} } + # Temporary workaround for handling arrayValue with PropertyFilter + if isinstance(self.value, Array): + rep['value'] = {'arrayValue': self.value.to_repr()} + else: + rep['value'] = self.value.to_repr() + return rep diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py index ce8e2ddc8..5208fe0b4 100644 --- a/datastore/tests/integration/smoke_test.py +++ b/datastore/tests/integration/smoke_test.py @@ -2,6 +2,7 @@ import pytest from gcloud.aio.auth import BUILD_GCLOUD_REST # pylint: disable=no-name-in-module +from gcloud.aio.datastore import Array from gcloud.aio.datastore import Datastore from gcloud.aio.datastore import Filter from gcloud.aio.datastore import GQLCursor @@ -272,6 +273,45 @@ async def test_query(creds: str, kind: str, project: str) -> None: assert len(after.entity_results) == num_results + 2 +@pytest.mark.asyncio +@pytest.mark.xfail(strict=False) +async def test_query_with_in_filter(creds: str, kind: str, project: str) -> None: + async with Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + property_filter = PropertyFilter( + prop='value', operator=PropertyFilterOperator.IN, + value=Array([Value(99), Value(100)]) + ) + query = Query(kind=kind, query_filter=Filter(property_filter)) + + before = await ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = await ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 99}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 100}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 101}, + ) + ] + await ds.commit(mutations, transaction=transaction, session=s) + + after = await ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 2 + + @pytest.mark.asyncio @pytest.mark.xfail(strict=False) async def test_gql_query(creds: str, kind: str, project: str) -> None: @@ -309,6 +349,42 @@ async def test_gql_query(creds: str, kind: str, project: str) -> None: after = await ds.runQuery(query, session=s) assert len(after.entity_results) == num_results + 3 +@pytest.mark.asyncio +@pytest.mark.xfail(strict=False) +async def test_gql_query_with_in_filter(creds: str, kind: str, project: str) -> None: + async with Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + query = GQLQuery( + f'SELECT * FROM {kind} WHERE value IN @values', + named_bindings={'values': Array([Value(99), Value(100)])}, + ) + + before = await ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = await ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 99}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 100}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 101}, + ), + ] + await ds.commit(mutations, transaction=transaction, session=s) + + after = await ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 2 @pytest.mark.asyncio @pytest.mark.xfail(strict=False) From 924d919d319839e74cce78c3f0f36c8f1c3200ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:11:34 +0000 Subject: [PATCH 2/6] refactor(lint): apply automatic lint fixes --- datastore/tests/integration/smoke_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py index 5208fe0b4..1fe006a1c 100644 --- a/datastore/tests/integration/smoke_test.py +++ b/datastore/tests/integration/smoke_test.py @@ -275,7 +275,8 @@ async def test_query(creds: str, kind: str, project: str) -> None: @pytest.mark.asyncio @pytest.mark.xfail(strict=False) -async def test_query_with_in_filter(creds: str, kind: str, project: str) -> None: +async def test_query_with_in_filter( + creds: str, kind: str, project: str) -> None: async with Session() as s: ds = Datastore(project=project, service_file=creds, session=s) @@ -349,9 +350,11 @@ async def test_gql_query(creds: str, kind: str, project: str) -> None: after = await ds.runQuery(query, session=s) assert len(after.entity_results) == num_results + 3 + @pytest.mark.asyncio @pytest.mark.xfail(strict=False) -async def test_gql_query_with_in_filter(creds: str, kind: str, project: str) -> None: +async def test_gql_query_with_in_filter( + creds: str, kind: str, project: str) -> None: async with Session() as s: ds = Datastore(project=project, service_file=creds, session=s) @@ -386,6 +389,7 @@ async def test_gql_query_with_in_filter(creds: str, kind: str, project: str) -> after = await ds.runQuery(query, session=s) assert len(after.entity_results) == num_results + 2 + @pytest.mark.asyncio @pytest.mark.xfail(strict=False) async def test_gql_query_pagination( From a4956b49c0579928856b6950cb4b31599ff31efb Mon Sep 17 00:00:00 2001 From: Onur Ozer Date: Wed, 12 Jun 2024 11:36:26 -0700 Subject: [PATCH 3/6] Formatting + adding NOT_IN filter support --- datastore/gcloud/aio/datastore/constants.py | 3 +- datastore/tests/integration/smoke_test.py | 77 +++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/datastore/gcloud/aio/datastore/constants.py b/datastore/gcloud/aio/datastore/constants.py index ab2497d20..c48e60a7c 100644 --- a/datastore/gcloud/aio/datastore/constants.py +++ b/datastore/gcloud/aio/datastore/constants.py @@ -49,11 +49,12 @@ class PropertyFilterOperator(enum.Enum): GREATER_THAN = 'GREATER_THAN' GREATER_THAN_OR_EQUAL = 'GREATER_THAN_OR_EQUAL' HAS_ANCESTOR = 'HAS_ANCESTOR' + IN = 'IN' LESS_THAN = 'LESS_THAN' LESS_THAN_OR_EQUAL = 'LESS_THAN_OR_EQUAL' NOT_EQUAL = 'NOT_EQUAL' + NOT_IN = 'NOT_IN' UNSPECIFIED = 'OPERATOR_UNSPECIFIED' - IN = 'IN' class ResultType(enum.Enum): diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py index 1fe006a1c..6c8dea799 100644 --- a/datastore/tests/integration/smoke_test.py +++ b/datastore/tests/integration/smoke_test.py @@ -313,6 +313,45 @@ async def test_query_with_in_filter( assert len(after.entity_results) == num_results + 2 +@pytest.mark.asyncio +@pytest.mark.xfail(strict=False) +async def test_query_with_not_in_filter(creds: str, kind: str, project: str) -> None: + async with Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + property_filter = PropertyFilter( + prop='value', operator=PropertyFilterOperator.NOT_IN, + value=Array([Value(99), Value(100), Value(30), Value(42)]) + ) + query = Query(kind=kind, query_filter=Filter(property_filter)) + + before = await ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = await ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 99}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 100}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 999}, + ) + ] + await ds.commit(mutations, transaction=transaction, session=s) + + after = await ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 1 + + @pytest.mark.asyncio @pytest.mark.xfail(strict=False) async def test_gql_query(creds: str, kind: str, project: str) -> None: @@ -390,6 +429,44 @@ async def test_gql_query_with_in_filter( assert len(after.entity_results) == num_results + 2 +@pytest.mark.asyncio +@pytest.mark.xfail(strict=False) +async def test_gql_query_with_not_in_filter(creds: str, kind: str, project: str) -> None: + async with Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + query = GQLQuery( + f'SELECT * FROM {kind} WHERE value NOT IN @values', + named_bindings={'values': Array([Value(30), Value(42), Value(99), Value(100)])}, + ) + + before = await ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = await ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 99}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 100}, + ), + ds.make_mutation( + Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 999}, + ), + ] + await ds.commit(mutations, transaction=transaction, session=s) + + after = await ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 1 + + @pytest.mark.asyncio @pytest.mark.xfail(strict=False) async def test_gql_query_pagination( From d7d6473f870969d3dd2441636f7e53490950724e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 18:40:32 +0000 Subject: [PATCH 4/6] refactor(lint): apply automatic lint fixes --- datastore/tests/integration/smoke_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py index 6c8dea799..6cfe94d92 100644 --- a/datastore/tests/integration/smoke_test.py +++ b/datastore/tests/integration/smoke_test.py @@ -315,7 +315,8 @@ async def test_query_with_in_filter( @pytest.mark.asyncio @pytest.mark.xfail(strict=False) -async def test_query_with_not_in_filter(creds: str, kind: str, project: str) -> None: +async def test_query_with_not_in_filter( + creds: str, kind: str, project: str) -> None: async with Session() as s: ds = Datastore(project=project, service_file=creds, session=s) @@ -431,13 +432,15 @@ async def test_gql_query_with_in_filter( @pytest.mark.asyncio @pytest.mark.xfail(strict=False) -async def test_gql_query_with_not_in_filter(creds: str, kind: str, project: str) -> None: +async def test_gql_query_with_not_in_filter( + creds: str, kind: str, project: str) -> None: async with Session() as s: ds = Datastore(project=project, service_file=creds, session=s) query = GQLQuery( f'SELECT * FROM {kind} WHERE value NOT IN @values', - named_bindings={'values': Array([Value(30), Value(42), Value(99), Value(100)])}, + named_bindings={'values': Array( + [Value(30), Value(42), Value(99), Value(100)])}, ) before = await ds.runQuery(query, session=s) From 975aaf13990df11c475d2aa1d1e8747dcfbf00ba Mon Sep 17 00:00:00 2001 From: Kevin James Date: Thu, 13 Jun 2024 18:12:02 +0100 Subject: [PATCH 5/6] chore(datastore): clean up some comments, fix lint --- datastore/gcloud/aio/datastore/constants.py | 1 - datastore/gcloud/aio/datastore/filter.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datastore/gcloud/aio/datastore/constants.py b/datastore/gcloud/aio/datastore/constants.py index c48e60a7c..729a8201a 100644 --- a/datastore/gcloud/aio/datastore/constants.py +++ b/datastore/gcloud/aio/datastore/constants.py @@ -44,7 +44,6 @@ class Operation(enum.Enum): class PropertyFilterOperator(enum.Enum): - # TODO: support IN / NOT_IN (requires rhs to be ArrayValue) EQUAL = 'EQUAL' GREATER_THAN = 'GREATER_THAN' GREATER_THAN_OR_EQUAL = 'GREATER_THAN_OR_EQUAL' diff --git a/datastore/gcloud/aio/datastore/filter.py b/datastore/gcloud/aio/datastore/filter.py index c9389460b..af2819847 100644 --- a/datastore/gcloud/aio/datastore/filter.py +++ b/datastore/gcloud/aio/datastore/filter.py @@ -91,7 +91,7 @@ class PropertyFilter(BaseFilter): def __init__( self, prop: str, operator: PropertyFilterOperator, - value: Union[Value, Array] + value: Union[Value, Array], ) -> None: self.prop = prop self.operator = operator @@ -115,11 +115,11 @@ def from_repr(cls, data: Dict[str, Any]) -> 'PropertyFilter': return cls(prop=prop, operator=operator, value=value) def to_repr(self) -> Dict[str, Any]: - rep = { + rep: Dict[str, Any] = { 'op': self.operator.value, - 'property': {'name': self.prop} + 'property': {'name': self.prop}, } - # Temporary workaround for handling arrayValue with PropertyFilter + # TODO: consider refactoring to look more like Value.to_repr() if isinstance(self.value, Array): rep['value'] = {'arrayValue': self.value.to_repr()} else: From 0cafcdd96e7cd2b6dcd4fe05e450e9e651cdf1e9 Mon Sep 17 00:00:00 2001 From: Kevin James Date: Thu, 13 Jun 2024 18:17:28 +0100 Subject: [PATCH 6/6] chore(datastore): further lint fixes --- datastore/tests/integration/smoke_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py index 6cfe94d92..959c3441e 100644 --- a/datastore/tests/integration/smoke_test.py +++ b/datastore/tests/integration/smoke_test.py @@ -282,7 +282,7 @@ async def test_query_with_in_filter( property_filter = PropertyFilter( prop='value', operator=PropertyFilterOperator.IN, - value=Array([Value(99), Value(100)]) + value=Array([Value(99), Value(100)]), ) query = Query(kind=kind, query_filter=Filter(property_filter)) @@ -305,7 +305,7 @@ async def test_query_with_in_filter( Operation.INSERT, Key(project, [PathElement(kind)]), properties={'value': 101}, - ) + ), ] await ds.commit(mutations, transaction=transaction, session=s) @@ -322,7 +322,7 @@ async def test_query_with_not_in_filter( property_filter = PropertyFilter( prop='value', operator=PropertyFilterOperator.NOT_IN, - value=Array([Value(99), Value(100), Value(30), Value(42)]) + value=Array([Value(99), Value(100), Value(30), Value(42)]), ) query = Query(kind=kind, query_filter=Filter(property_filter)) @@ -345,7 +345,7 @@ async def test_query_with_not_in_filter( Operation.INSERT, Key(project, [PathElement(kind)]), properties={'value': 999}, - ) + ), ] await ds.commit(mutations, transaction=transaction, session=s)