From 4701b7aea7b93160982ff0ec40fdfebe0e87f63b Mon Sep 17 00:00:00 2001 From: Jeronimo Date: Wed, 12 Jun 2024 10:24:04 -0300 Subject: [PATCH 01/12] Adds support for enabling and disabling async replication in schema. This can be used `ONLY` with Weaviate Server ``v1.26.0`` or later. This can be configured in class schema like this: .. code-block:: python my_class = { "class": "MyClass", ..., "replicationConfig": { "factor": 1, "async_enabled": true } } Signed-off-by: Jeronimo test: ajust test cases to support asyncRep Signed-off-by: Jeronimo chore: use snake case for async replication Signed-off-by: Jeronimo test: ajust test cases to support asyncRep Signed-off-by: Jeronimo chore: handle optional async_enabled attribute Signed-off-by: Jeronimo chore: add missing colon in payload Signed-off-by: Jeronimo chore: linter changes Signed-off-by: Jeronimo test: parametrize replication config Signed-off-by: Jeronimo test: parametrize replication config Signed-off-by: Jeronimo --- integration/test_collection_config.py | 8 +++++--- integration/test_collection_multi_node.py | 2 +- integration_v3/test_graphql.py | 2 +- integration_v3/test_schema.py | 9 +++++++-- mock_tests/test_collection.py | 4 ++-- weaviate/collections/classes/config.py | 19 +++++++++++++++---- .../collections/classes/config_methods.py | 5 ++++- weaviate/schema/crud_schema.py | 6 ++++-- 8 files changed, 39 insertions(+), 16 deletions(-) diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index 7c099734d..e13afe3b1 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -136,6 +136,7 @@ def test_collection_config_empty(collection_factory: CollectionFactory) -> None: assert config.multi_tenancy_config.enabled is False assert config.replication_config.factor == 1 + assert config.replication_config.async_enabled is False assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert config.vector_index_config.cleanup_interval_seconds == 300 @@ -189,6 +190,7 @@ def test_collection_config_defaults(collection_factory: CollectionFactory) -> No assert config.multi_tenancy_config.enabled is True assert config.replication_config.factor == 1 + assert config.replication_config.async_enabled is False assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert config.vector_index_config.cleanup_interval_seconds == 300 @@ -241,7 +243,7 @@ def test_collection_config_full(collection_factory: CollectionFactory) -> None: multi_tenancy_config=Configure.multi_tenancy( enabled=True, auto_tenant_activation=True, auto_tenant_creation=True ), - # replication_config=Configure.replication(factor=2), # currently not updateable in RAFT + # replication_config=Configure.replication(factor=2, async_enabled=True), # currently not updateable in RAFT vector_index_config=Configure.VectorIndex.hnsw( cleanup_interval_seconds=10, distance_metric=VectorDistances.DOT, @@ -373,7 +375,7 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None stopwords_preset=StopwordsPreset.EN, stopwords_removals=["the"], ), - # replication_config=Reconfigure.replication(factor=2), # currently not updateable in RAFT + # replication_config=Reconfigure.replication(factor=2, async_enabled=True), # currently not updateable in RAFT vectorizer_config=Reconfigure.VectorIndex.hnsw( vector_cache_max_objects=2000000, quantizer=Reconfigure.VectorIndex.Quantizer.pq( @@ -711,7 +713,7 @@ def test_config_export_and_recreate_from_dict(collection_factory: CollectionFact Property(name="age", data_type=DataType.INT), ], multi_tenancy_config=Configure.multi_tenancy(enabled=True), - replication_config=Configure.replication(factor=1), + replication_config=Configure.replication(factor=1, async_enabled=False), vector_index_config=Configure.VectorIndex.hnsw( quantizer=Configure.VectorIndex.Quantizer.pq(centroids=256) ), diff --git a/integration/test_collection_multi_node.py b/integration/test_collection_multi_node.py index 012c244f5..6765f0809 100644 --- a/integration/test_collection_multi_node.py +++ b/integration/test_collection_multi_node.py @@ -22,7 +22,7 @@ def test_consistency_on_multinode( properties=[ Property(name="name", data_type=DataType.TEXT), ], - replication_config=Configure.replication(factor=2), + replication_config=Configure.replication(factor=2, async_enabled=False), ports=(8087, 50058), ).with_consistency_level(level) diff --git a/integration_v3/test_graphql.py b/integration_v3/test_graphql.py index 74c85ef12..aaacb45e7 100644 --- a/integration_v3/test_graphql.py +++ b/integration_v3/test_graphql.py @@ -94,7 +94,7 @@ def client(request): if opts.get("cluster"): port = 8087 for _, c in enumerate(schema["classes"]): - c["replicationConfig"] = {"factor": 2} + c["replicationConfig"] = {"factor": 2, "async_enabled": False} client = weaviate.Client(f"http://localhost:{port}") client.schema.delete_all() diff --git a/integration_v3/test_schema.py b/integration_v3/test_schema.py index e6f7278f5..a52502ff0 100644 --- a/integration_v3/test_schema.py +++ b/integration_v3/test_schema.py @@ -15,8 +15,9 @@ def client(): @pytest.mark.parametrize("replicationFactor", [None, 1]) -def test_create_class_with_implicit_and_explicit_replication_factor( - client: weaviate.Client, replicationFactor: Optional[int] +@pytest.mark.parametrize("asyncEnabled", [None, False]) +def test_create_class_with_implicit_and_explicit_replication_config( + client: weaviate.Client, replicationFactor: Optional[int], asyncEnabled: Optional[bool] ): single_class = { "class": "Barbecue", @@ -31,16 +32,20 @@ def test_create_class_with_implicit_and_explicit_replication_factor( } if replicationFactor is None: expected_factor = 1 + if asyncEnabled is None: + expected_async_enabled = False else: expected_factor = replicationFactor single_class["replicationConfig"] = { "factor": replicationFactor, + "async_enabled": asyncEnabled, } client.schema.create_class(single_class) created_class = client.schema.get("Barbecue") assert created_class["class"] == "Barbecue" assert created_class["replicationConfig"]["factor"] == expected_factor + assert created_class["replicationConfig"].get("async_enabled") == expected_async_enabled client.schema.delete_class("Barbecue") diff --git a/mock_tests/test_collection.py b/mock_tests/test_collection.py index ccc713b1f..97dcda1f7 100644 --- a/mock_tests/test_collection.py +++ b/mock_tests/test_collection.py @@ -180,7 +180,7 @@ def test_missing_multi_tenancy_config( ), properties=[], references=[], - replication_config=ReplicationConfig(factor=0), + replication_config=ReplicationConfig(factor=0, async_enabled=False), vector_index_config=vic, vector_index_type=VectorIndexType.FLAT, vectorizer=Vectorizers.NONE, @@ -242,7 +242,7 @@ def test_return_from_bind_module( "invertedIndexConfig": ii_config, "multiTenancyConfig": config.multi_tenancy()._to_dict(), "vectorizer": "multi2vec-bind", - "replicationConfig": {"factor": 2}, + "replicationConfig": {"factor": 2, "async_enabled": False}, "moduleConfig": {"multi2vec-bind": {}}, } weaviate_auth_mock.expect_request("/v1/schema/TestBindCollection").respond_with_json( diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index e0b7700e3..6d8693f39 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -310,10 +310,12 @@ class _ShardingConfigCreate(_ConfigCreateModel): class _ReplicationConfigCreate(_ConfigCreateModel): factor: Optional[int] + async_enabled: Optional[bool] class _ReplicationConfigUpdate(_ConfigUpdateModel): factor: Optional[int] + async_enabled: Optional[bool] class _BM25ConfigCreate(_ConfigCreateModel): @@ -1053,6 +1055,7 @@ def to_dict(self) -> Dict[str, Any]: @dataclass class _ReplicationConfig(_ConfigBase): factor: int + async_enabled: bool ReplicationConfig = _ReplicationConfig @@ -1775,14 +1778,18 @@ def multi_tenancy( ) @staticmethod - def replication(factor: Optional[int] = None) -> _ReplicationConfigCreate: + def replication( + factor: Optional[int] = None, async_enabled: Optional[bool] = None + ) -> _ReplicationConfigCreate: """Create a `ReplicationConfigCreate` object to be used when defining the replication configuration of Weaviate. Arguments: `factor` The replication factor. + `async_enabled` + Enabled async replication. """ - return _ReplicationConfigCreate(factor=factor) + return _ReplicationConfigCreate(factor=factor, async_enabled=async_enabled) @staticmethod def sharding( @@ -1977,7 +1984,9 @@ def inverted_index( ) @staticmethod - def replication(factor: Optional[int] = None) -> _ReplicationConfigUpdate: + def replication( + factor: Optional[int] = None, async_enabled: Optional[bool] = None + ) -> _ReplicationConfigUpdate: """Create a `ReplicationConfigUpdate` object. Use this method when defining the `replication_config` argument in `collection.update()`. @@ -1985,8 +1994,10 @@ def replication(factor: Optional[int] = None) -> _ReplicationConfigUpdate: Arguments: `factor` The replication factor. + `async_enabled` + Enable async replication. """ - return _ReplicationConfigUpdate(factor=factor) + return _ReplicationConfigUpdate(factor=factor, async_enabled=async_enabled) @staticmethod def multi_tenancy( diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index 9e97188aa..2d3aac957 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -251,7 +251,10 @@ def _collection_config_from_json(schema: Dict[str, Any]) -> _CollectionConfig: ), properties=_properties_from_config(schema) if schema.get("properties") is not None else [], references=_references_from_config(schema) if schema.get("properties") is not None else [], - replication_config=_ReplicationConfig(factor=schema["replicationConfig"]["factor"]), + replication_config=_ReplicationConfig( + factor=schema["replicationConfig"]["factor"], + async_enabled=schema["replicationConfig"].get("async_enabled"), + ), reranker_config=__get_rerank_config(schema), sharding_config=( None diff --git a/weaviate/schema/crud_schema.py b/weaviate/schema/crud_schema.py index 29650194a..6da4b9236 100644 --- a/weaviate/schema/crud_schema.py +++ b/weaviate/schema/crud_schema.py @@ -516,7 +516,8 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorIndexType": "hnsw", "vectorizer": "text2vec-contextionary", "replicationConfig": { - "factor": 1 + "factor": 1, + "async_enabled": false } } ] @@ -545,7 +546,8 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorIndexType": "hnsw", "vectorizer": "text2vec-contextionary", "replicationConfig": { - "factor": 1 + "factor": 1, + "async_enabled": false } } From 51afc997dded523399bca8ac1cc5c00c81a367d5 Mon Sep 17 00:00:00 2001 From: Jose Luis Franco Arza Date: Tue, 18 Jun 2024 15:00:31 +0200 Subject: [PATCH 02/12] References to async_enabled should be asyncEnabled. This commit fixes some misreferences to the asyncEnabled property which were using async_enabled instead. --- integration_v3/test_graphql.py | 2 +- integration_v3/test_schema.py | 19 ++++++++----------- mock_tests/test_collection.py | 2 +- weaviate/collections/classes/config.py | 8 ++++---- .../collections/classes/config_methods.py | 2 +- weaviate/schema/crud_schema.py | 4 ++-- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/integration_v3/test_graphql.py b/integration_v3/test_graphql.py index aaacb45e7..2e77cfe09 100644 --- a/integration_v3/test_graphql.py +++ b/integration_v3/test_graphql.py @@ -94,7 +94,7 @@ def client(request): if opts.get("cluster"): port = 8087 for _, c in enumerate(schema["classes"]): - c["replicationConfig"] = {"factor": 2, "async_enabled": False} + c["replicationConfig"] = {"factor": 2, "asyncEnabled": False} client = weaviate.Client(f"http://localhost:{port}") client.schema.delete_all() diff --git a/integration_v3/test_schema.py b/integration_v3/test_schema.py index a52502ff0..58ba7bd58 100644 --- a/integration_v3/test_schema.py +++ b/integration_v3/test_schema.py @@ -30,22 +30,19 @@ def test_create_class_with_implicit_and_explicit_replication_config( }, ], } - if replicationFactor is None: - expected_factor = 1 - if asyncEnabled is None: - expected_async_enabled = False - else: - expected_factor = replicationFactor - single_class["replicationConfig"] = { - "factor": replicationFactor, - "async_enabled": asyncEnabled, - } + + expected_factor = 1 if replicationFactor == None else replicationFactor + expected_async_enabled = False if asyncEnabled == None else asyncEnabled + single_class["replicationConfig"] = { + "factor": replicationFactor, + "asyncEnabled": asyncEnabled, + } client.schema.create_class(single_class) created_class = client.schema.get("Barbecue") assert created_class["class"] == "Barbecue" assert created_class["replicationConfig"]["factor"] == expected_factor - assert created_class["replicationConfig"].get("async_enabled") == expected_async_enabled + assert created_class["replicationConfig"].get("asyncEnabled", False) == expected_async_enabled client.schema.delete_class("Barbecue") diff --git a/mock_tests/test_collection.py b/mock_tests/test_collection.py index 97dcda1f7..4eab6187c 100644 --- a/mock_tests/test_collection.py +++ b/mock_tests/test_collection.py @@ -242,7 +242,7 @@ def test_return_from_bind_module( "invertedIndexConfig": ii_config, "multiTenancyConfig": config.multi_tenancy()._to_dict(), "vectorizer": "multi2vec-bind", - "replicationConfig": {"factor": 2, "async_enabled": False}, + "replicationConfig": {"factor": 2, "asyncEnabled": False}, "moduleConfig": {"multi2vec-bind": {}}, } weaviate_auth_mock.expect_request("/v1/schema/TestBindCollection").respond_with_json( diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index 6d8693f39..169741d89 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -310,12 +310,12 @@ class _ShardingConfigCreate(_ConfigCreateModel): class _ReplicationConfigCreate(_ConfigCreateModel): factor: Optional[int] - async_enabled: Optional[bool] + asyncEnabled: Optional[bool] class _ReplicationConfigUpdate(_ConfigUpdateModel): factor: Optional[int] - async_enabled: Optional[bool] + asyncEnabled: Optional[bool] class _BM25ConfigCreate(_ConfigCreateModel): @@ -1789,7 +1789,7 @@ def replication( `async_enabled` Enabled async replication. """ - return _ReplicationConfigCreate(factor=factor, async_enabled=async_enabled) + return _ReplicationConfigCreate(factor=factor, asyncEnabled=async_enabled) @staticmethod def sharding( @@ -1997,7 +1997,7 @@ def replication( `async_enabled` Enable async replication. """ - return _ReplicationConfigUpdate(factor=factor, async_enabled=async_enabled) + return _ReplicationConfigUpdate(factor=factor, asyncEnabled=async_enabled) @staticmethod def multi_tenancy( diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index 2d3aac957..d58e1de3e 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -253,7 +253,7 @@ def _collection_config_from_json(schema: Dict[str, Any]) -> _CollectionConfig: references=_references_from_config(schema) if schema.get("properties") is not None else [], replication_config=_ReplicationConfig( factor=schema["replicationConfig"]["factor"], - async_enabled=schema["replicationConfig"].get("async_enabled"), + async_enabled=schema["replicationConfig"]["asyncEnabled"], ), reranker_config=__get_rerank_config(schema), sharding_config=( diff --git a/weaviate/schema/crud_schema.py b/weaviate/schema/crud_schema.py index 6da4b9236..b9accc1f5 100644 --- a/weaviate/schema/crud_schema.py +++ b/weaviate/schema/crud_schema.py @@ -517,7 +517,7 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorizer": "text2vec-contextionary", "replicationConfig": { "factor": 1, - "async_enabled": false + "asyncEnabled": false } } ] @@ -547,7 +547,7 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorizer": "text2vec-contextionary", "replicationConfig": { "factor": 1, - "async_enabled": false + "asyncEnabled": false } } From 9a8c9572e5ef270beae531733564803b4aeed1e5 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 11:45:46 +0100 Subject: [PATCH 03/12] Make changes in response to review --- .github/workflows/main.yaml | 36 +++++++++---------- integration/test_collection_config.py | 24 ++++++++----- integration_v3/test_graphql.py | 2 +- integration_v3/test_schema.py | 16 ++++----- weaviate/collections/classes/config.py | 2 ++ .../collections/classes/config_methods.py | 2 +- weaviate/schema/crud_schema.py | 2 -- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c8272552f..852744d95 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,18 @@ on: env: WEAVIATE_123: 1.23.14 WEAVIATE_124: 1.24.10 - WEAVIATE_125: preview--c57fecf + WEAVIATE_125: 1.25.5 + WEAVIATE_126: 1.26.0-preview0-b97790d + INTEGRATION_TEST_VERSIONS: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126} + { py: "3.12", weaviate: $WEAVIATE_126} + ] jobs: @@ -120,15 +131,7 @@ jobs: strategy: fail-fast: false matrix: - versions: [ - { py: "3.8", weaviate: $WEAVIATE_125}, - { py: "3.9", weaviate: $WEAVIATE_125}, - { py: "3.10", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.12", weaviate: $WEAVIATE_125} - ] + versions: $INTEGRATION_TEST_VERSIONS optional_dependencies: [false] steps: - uses: actions/checkout@v4 @@ -171,15 +174,7 @@ jobs: strategy: fail-fast: false matrix: - versions: [ - { py: "3.8", weaviate: $WEAVIATE_125}, - { py: "3.9", weaviate: $WEAVIATE_125}, - { py: "3.10", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.12", weaviate: $WEAVIATE_125} - ] + versions: $INTEGRATION_TEST_VERSIONS optional_dependencies: [false] steps: - uses: actions/checkout@v4 @@ -284,7 +279,8 @@ jobs: server: [ $WEAVIATE_123, $WEAVIATE_124, - $WEAVIATE_125 + $WEAVIATE_125, + $WEAVIATE_126 ] steps: - name: Download build artifact to append to release diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index e13afe3b1..9117d11c5 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -1,6 +1,6 @@ from typing import Generator -import pytest as pytest +import pytest import weaviate from integration.conftest import OpenAICollection, CollectionFactory @@ -243,7 +243,7 @@ def test_collection_config_full(collection_factory: CollectionFactory) -> None: multi_tenancy_config=Configure.multi_tenancy( enabled=True, auto_tenant_activation=True, auto_tenant_creation=True ), - # replication_config=Configure.replication(factor=2, async_enabled=True), # currently not updateable in RAFT + replication_config=Configure.replication(factor=2, async_enabled=True), vector_index_config=Configure.VectorIndex.hnsw( cleanup_interval_seconds=10, distance_metric=VectorDistances.DOT, @@ -314,11 +314,18 @@ def test_collection_config_full(collection_factory: CollectionFactory) -> None: assert config.multi_tenancy_config.enabled is True if collection._connection._weaviate_version.is_at_least(1, 25, 0): assert config.multi_tenancy_config.auto_tenant_activation is True - # change to 1.25.2 after it is out + else: + assert config.multi_tenancy_config.auto_tenant_activation is False if collection._connection._weaviate_version.is_at_least(1, 25, patch=1): assert config.multi_tenancy_config.auto_tenant_creation is True + else: + assert config.multi_tenancy_config.auto_tenant_creation is False - # assert config.replication_config.factor == 2 + assert config.replication_config.factor == 2 + if collection._connection._weaviate_version.is_at_least(1, 26, 0): + assert config.replication_config.async_enabled is True + else: + assert config.replication_config.async_enabled is False assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert isinstance(config.vector_index_config.quantizer, _PQConfig) @@ -393,7 +400,7 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None config = collection.config.get() - # assert config.description == "Test" + assert config.description == "Test" assert config.inverted_index_config.bm25.b == 0.8 assert config.inverted_index_config.bm25.k1 == 1.25 @@ -402,6 +409,7 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None assert config.inverted_index_config.stopwords.removals == ["the"] # assert config.replication_config.factor == 2 + # assert config.replication_config.async_enabled is True assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert isinstance(config.vector_index_config.quantizer, _PQConfig) @@ -641,14 +649,12 @@ def test_config_vector_index_hnsw_and_quantizer_pq(collection_factory: Collectio ], ) def test_config_reranker_module( - client: weaviate.WeaviateClient, + collection_factory: CollectionFactory, reranker_config: _RerankerConfigCreate, expected_reranker: Rerankers, expected_model: dict, ) -> None: - client.collections.delete("TestCollectionConfigRerankerModule") - collection = client.collections.create( - name="TestCollectionConfigRerankerModule", + collection = collection_factory( reranker_config=reranker_config, vectorizer_config=Configure.Vectorizer.none(), ) diff --git a/integration_v3/test_graphql.py b/integration_v3/test_graphql.py index 2e77cfe09..74c85ef12 100644 --- a/integration_v3/test_graphql.py +++ b/integration_v3/test_graphql.py @@ -94,7 +94,7 @@ def client(request): if opts.get("cluster"): port = 8087 for _, c in enumerate(schema["classes"]): - c["replicationConfig"] = {"factor": 2, "asyncEnabled": False} + c["replicationConfig"] = {"factor": 2} client = weaviate.Client(f"http://localhost:{port}") client.schema.delete_all() diff --git a/integration_v3/test_schema.py b/integration_v3/test_schema.py index 58ba7bd58..1a19701e2 100644 --- a/integration_v3/test_schema.py +++ b/integration_v3/test_schema.py @@ -15,7 +15,6 @@ def client(): @pytest.mark.parametrize("replicationFactor", [None, 1]) -@pytest.mark.parametrize("asyncEnabled", [None, False]) def test_create_class_with_implicit_and_explicit_replication_config( client: weaviate.Client, replicationFactor: Optional[int], asyncEnabled: Optional[bool] ): @@ -30,19 +29,18 @@ def test_create_class_with_implicit_and_explicit_replication_config( }, ], } - - expected_factor = 1 if replicationFactor == None else replicationFactor - expected_async_enabled = False if asyncEnabled == None else asyncEnabled - single_class["replicationConfig"] = { - "factor": replicationFactor, - "asyncEnabled": asyncEnabled, - } + if replicationFactor is None: + expected_factor = 1 + else: + expected_factor = replicationFactor + single_class["replicationConfig"] = { + "factor": replicationFactor, + } client.schema.create_class(single_class) created_class = client.schema.get("Barbecue") assert created_class["class"] == "Barbecue" assert created_class["replicationConfig"]["factor"] == expected_factor - assert created_class["replicationConfig"].get("asyncEnabled", False) == expected_async_enabled client.schema.delete_class("Barbecue") diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index 169741d89..ebe1c2cc7 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -1783,6 +1783,8 @@ def replication( ) -> _ReplicationConfigCreate: """Create a `ReplicationConfigCreate` object to be used when defining the replication configuration of Weaviate. + NOTE: `async_enabled` is only available with WeaviateDB `>=v1.26.0` + Arguments: `factor` The replication factor. diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index d58e1de3e..44b3490ca 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -253,7 +253,7 @@ def _collection_config_from_json(schema: Dict[str, Any]) -> _CollectionConfig: references=_references_from_config(schema) if schema.get("properties") is not None else [], replication_config=_ReplicationConfig( factor=schema["replicationConfig"]["factor"], - async_enabled=schema["replicationConfig"]["asyncEnabled"], + async_enabled=schema["replicationConfig"].get("asyncEnabled", False), ), reranker_config=__get_rerank_config(schema), sharding_config=( diff --git a/weaviate/schema/crud_schema.py b/weaviate/schema/crud_schema.py index b9accc1f5..f544a28c5 100644 --- a/weaviate/schema/crud_schema.py +++ b/weaviate/schema/crud_schema.py @@ -517,7 +517,6 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorizer": "text2vec-contextionary", "replicationConfig": { "factor": 1, - "asyncEnabled": false } } ] @@ -547,7 +546,6 @@ def get(self, class_name: Optional[str] = None) -> dict: "vectorizer": "text2vec-contextionary", "replicationConfig": { "factor": 1, - "asyncEnabled": false } } From 388aef331ebeffd3feb8b6706fb5900b067cba93 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 11:55:38 +0100 Subject: [PATCH 04/12] Fix yaml syntax error --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 852744d95..70ffdec4d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -17,7 +17,7 @@ env: WEAVIATE_123: 1.23.14 WEAVIATE_124: 1.24.10 WEAVIATE_125: 1.25.5 - WEAVIATE_126: 1.26.0-preview0-b97790d + WEAVIATE_126: "1.26.0-preview0-b97790d" INTEGRATION_TEST_VERSIONS: [ { py: "3.8", weaviate: $WEAVIATE_126}, { py: "3.9", weaviate: $WEAVIATE_126}, From c1ccb95abfe5fecd9aa2a1ec43694a17849e5686 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 11:58:13 +0100 Subject: [PATCH 05/12] Undo addition of INTEGRATION_TEST_VERSIONS env var --- .github/workflows/main.yaml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 70ffdec4d..0e43622db 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -17,17 +17,7 @@ env: WEAVIATE_123: 1.23.14 WEAVIATE_124: 1.24.10 WEAVIATE_125: 1.25.5 - WEAVIATE_126: "1.26.0-preview0-b97790d" - INTEGRATION_TEST_VERSIONS: [ - { py: "3.8", weaviate: $WEAVIATE_126}, - { py: "3.9", weaviate: $WEAVIATE_126}, - { py: "3.10", weaviate: $WEAVIATE_126}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_126} - { py: "3.12", weaviate: $WEAVIATE_126} - ] + WEAVIATE_126: 1.26.0-preview0-b97790d jobs: @@ -131,7 +121,16 @@ jobs: strategy: fail-fast: false matrix: - versions: $INTEGRATION_TEST_VERSIONS + versions: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126} + { py: "3.12", weaviate: $WEAVIATE_126} + ] optional_dependencies: [false] steps: - uses: actions/checkout@v4 @@ -174,7 +173,16 @@ jobs: strategy: fail-fast: false matrix: - versions: $INTEGRATION_TEST_VERSIONS + versions: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126} + { py: "3.12", weaviate: $WEAVIATE_126} + ] optional_dependencies: [false] steps: - uses: actions/checkout@v4 From d2c24fcbfc60a12a25cc1f77520b463650ac510b Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 11:59:06 +0100 Subject: [PATCH 06/12] Add missing comma in list --- .github/workflows/main.yaml | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0e43622db..73b6d7469 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,6 +18,16 @@ env: WEAVIATE_124: 1.24.10 WEAVIATE_125: 1.25.5 WEAVIATE_126: 1.26.0-preview0-b97790d + INTEGRATION_TEST_VERSIONS: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126}, + { py: "3.12", weaviate: $WEAVIATE_126} + ] jobs: @@ -121,16 +131,7 @@ jobs: strategy: fail-fast: false matrix: - versions: [ - { py: "3.8", weaviate: $WEAVIATE_126}, - { py: "3.9", weaviate: $WEAVIATE_126}, - { py: "3.10", weaviate: $WEAVIATE_126}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_126} - { py: "3.12", weaviate: $WEAVIATE_126} - ] + versions: $INTEGRATION_TEST_VERSIONS optional_dependencies: [false] steps: - uses: actions/checkout@v4 @@ -173,16 +174,7 @@ jobs: strategy: fail-fast: false matrix: - versions: [ - { py: "3.8", weaviate: $WEAVIATE_126}, - { py: "3.9", weaviate: $WEAVIATE_126}, - { py: "3.10", weaviate: $WEAVIATE_126}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_126} - { py: "3.12", weaviate: $WEAVIATE_126} - ] + versions: $INTEGRATION_TEST_VERSIONS optional_dependencies: [false] steps: - uses: actions/checkout@v4 From b7437ca6d1999118b98d937b742b23684d46d2f8 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 12:00:32 +0100 Subject: [PATCH 07/12] Sequence not allowed in env vars --- .github/workflows/main.yaml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 73b6d7469..3937aaedb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,16 +18,6 @@ env: WEAVIATE_124: 1.24.10 WEAVIATE_125: 1.25.5 WEAVIATE_126: 1.26.0-preview0-b97790d - INTEGRATION_TEST_VERSIONS: [ - { py: "3.8", weaviate: $WEAVIATE_126}, - { py: "3.9", weaviate: $WEAVIATE_126}, - { py: "3.10", weaviate: $WEAVIATE_126}, - { py: "3.11", weaviate: $WEAVIATE_123}, - { py: "3.11", weaviate: $WEAVIATE_124}, - { py: "3.11", weaviate: $WEAVIATE_125}, - { py: "3.11", weaviate: $WEAVIATE_126}, - { py: "3.12", weaviate: $WEAVIATE_126} - ] jobs: @@ -131,7 +121,16 @@ jobs: strategy: fail-fast: false matrix: - versions: $INTEGRATION_TEST_VERSIONS + versions: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126}, + { py: "3.12", weaviate: $WEAVIATE_126} + ] optional_dependencies: [false] steps: - uses: actions/checkout@v4 @@ -174,7 +173,16 @@ jobs: strategy: fail-fast: false matrix: - versions: $INTEGRATION_TEST_VERSIONS + versions: [ + { py: "3.8", weaviate: $WEAVIATE_126}, + { py: "3.9", weaviate: $WEAVIATE_126}, + { py: "3.10", weaviate: $WEAVIATE_126}, + { py: "3.11", weaviate: $WEAVIATE_123}, + { py: "3.11", weaviate: $WEAVIATE_124}, + { py: "3.11", weaviate: $WEAVIATE_125}, + { py: "3.11", weaviate: $WEAVIATE_126}, + { py: "3.12", weaviate: $WEAVIATE_126} + ] optional_dependencies: [false] steps: - uses: actions/checkout@v4 From c0b40db365e512e7fb4be9b70713a2f9176b7ba9 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 26 Jun 2024 16:58:19 +0100 Subject: [PATCH 08/12] Update CI to use latest working image --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0a5a4bb77..189c14bce 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -17,7 +17,7 @@ env: WEAVIATE_123: 1.23.15 WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 - WEAVIATE_126: 1.26.0-preview0-b97790d + WEAVIATE_126: preview-replace-if-targetvectors-nil-with-if-vectorsearch-when-extracting-metadata-certainty-ea8d536 jobs: From e7e1d8c2ce8d6586bc602d74b39936f237b96e6e Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 27 Jun 2024 09:45:09 +0100 Subject: [PATCH 09/12] Remove redundant arg from v3 test --- integration_v3/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_v3/test_schema.py b/integration_v3/test_schema.py index 1a19701e2..5b89ab627 100644 --- a/integration_v3/test_schema.py +++ b/integration_v3/test_schema.py @@ -16,7 +16,7 @@ def client(): @pytest.mark.parametrize("replicationFactor", [None, 1]) def test_create_class_with_implicit_and_explicit_replication_config( - client: weaviate.Client, replicationFactor: Optional[int], asyncEnabled: Optional[bool] + client: weaviate.Client, replicationFactor: Optional[int] ): single_class = { "class": "Barbecue", From 06f2bc09fa5c6acc9b2ba79f065576b5ba709194 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 27 Jun 2024 12:03:26 +0100 Subject: [PATCH 10/12] Update to latest image on main --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 189c14bce..1cbffcc59 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -17,7 +17,7 @@ env: WEAVIATE_123: 1.23.15 WEAVIATE_124: 1.24.19 WEAVIATE_125: 1.25.5 - WEAVIATE_126: preview-replace-if-targetvectors-nil-with-if-vectorsearch-when-extracting-metadata-certainty-ea8d536 + WEAVIATE_126: 1.26.0-preview0-87c34a9 jobs: From e3108f12e9e95969ed12832676deaf5c72796bc2 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 27 Jun 2024 12:04:07 +0100 Subject: [PATCH 11/12] Align config capatabilities between different versions and add back commented code --- integration/test_collection_config.py | 52 ++++++++++++++++++--------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index 9117d11c5..4b8d899ad 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -312,11 +312,11 @@ def test_collection_config_full(collection_factory: CollectionFactory) -> None: assert config.inverted_index_config.stopwords.removals == ["the"] assert config.multi_tenancy_config.enabled is True - if collection._connection._weaviate_version.is_at_least(1, 25, 0): + if collection._connection._weaviate_version.is_at_least(1, 25, 2): assert config.multi_tenancy_config.auto_tenant_activation is True else: assert config.multi_tenancy_config.auto_tenant_activation is False - if collection._connection._weaviate_version.is_at_least(1, 25, patch=1): + if collection._connection._weaviate_version.is_at_least(1, 25, 1): assert config.multi_tenancy_config.auto_tenant_creation is True else: assert config.multi_tenancy_config.auto_tenant_creation is False @@ -365,12 +365,10 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None config = collection.config.get() assert config.replication_config.factor == 1 + assert config.replication_config.async_enabled is False assert config.multi_tenancy_config.enabled is True - if collection._connection._weaviate_version.is_at_least(1, 25, 0): - assert config.multi_tenancy_config.auto_tenant_activation is False - # change to 1.25.2 after it is out - if collection._connection._weaviate_version.is_at_least(1, 25, patch=1): - assert config.multi_tenancy_config.auto_tenant_creation is False + assert config.multi_tenancy_config.auto_tenant_activation is False + assert config.multi_tenancy_config.auto_tenant_creation is False collection.config.update( description="Test", @@ -382,7 +380,9 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None stopwords_preset=StopwordsPreset.EN, stopwords_removals=["the"], ), - # replication_config=Reconfigure.replication(factor=2, async_enabled=True), # currently not updateable in RAFT + replication_config=Reconfigure.replication( + factor=2, async_enabled=True + ), # currently not updateable in RAFT vectorizer_config=Reconfigure.VectorIndex.hnsw( vector_cache_max_objects=2000000, quantizer=Reconfigure.VectorIndex.Quantizer.pq( @@ -400,7 +400,10 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None config = collection.config.get() - assert config.description == "Test" + if collection._connection._weaviate_version.is_at_least(1, 25, 2): + assert config.description == "Test" + else: + assert config.description is None assert config.inverted_index_config.bm25.b == 0.8 assert config.inverted_index_config.bm25.k1 == 1.25 @@ -408,8 +411,12 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None # assert config.inverted_index_config.stopwords.additions is ["a"] # potential weaviate bug, this returns as None assert config.inverted_index_config.stopwords.removals == ["the"] - # assert config.replication_config.factor == 2 - # assert config.replication_config.async_enabled is True + assert config.replication_config.factor == 2 + + if collection._connection._weaviate_version.is_at_least(1, 26, 0): + assert config.replication_config.async_enabled is True + else: + assert config.replication_config.async_enabled is False assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert isinstance(config.vector_index_config.quantizer, _PQConfig) @@ -434,11 +441,16 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None assert config.vector_index_type == VectorIndexType.HNSW assert config.multi_tenancy_config.enabled is True - if collection._connection._weaviate_version.is_at_least(1, 25, 0): + + if collection._connection._weaviate_version.is_at_least(1, 25, 2): assert config.multi_tenancy_config.auto_tenant_activation is True - # change to 1.25.2 after it is out - if collection._connection._weaviate_version.is_at_least(1, 25, patch=1): + else: + assert config.multi_tenancy_config.auto_tenant_activation is False + + if collection._connection._weaviate_version.is_at_least(1, 25, 1): assert config.multi_tenancy_config.auto_tenant_creation is True + else: + assert config.multi_tenancy_config.auto_tenant_creation is False collection.config.update( vectorizer_config=Reconfigure.VectorIndex.hnsw( @@ -447,7 +459,10 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None ) config = collection.config.get() - # assert config.description == "Test" + if collection._connection._weaviate_version.is_at_least(1, 25, 2): + assert config.description == "Test" + else: + assert config.description is None assert config.inverted_index_config.bm25.b == 0.8 assert config.inverted_index_config.bm25.k1 == 1.25 @@ -455,7 +470,12 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None # assert config.inverted_index_config.stopwords.additions is ["a"] # potential weaviate bug, this returns as None assert config.inverted_index_config.stopwords.removals == ["the"] - # assert config.replication_config.factor == 2 + assert config.replication_config.factor == 2 + + if collection._connection._weaviate_version.is_at_least(1, 26, 0): + assert config.replication_config.async_enabled is True + else: + assert config.replication_config.async_enabled is False assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) assert config.vector_index_config.cleanup_interval_seconds == 300 From 9ce3269db6a5af878b91581a7782d384fe2ed4d7 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 27 Jun 2024 12:42:47 +0100 Subject: [PATCH 12/12] Fix weaviate version check for lower vers --- integration/test_collection_config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index 4b8d899ad..5ac9299ad 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -400,7 +400,9 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None config = collection.config.get() - if collection._connection._weaviate_version.is_at_least(1, 25, 2): + if collection._connection._weaviate_version.is_at_least( + 1, 25, 2 + ) or collection._connection._weaviate_version.is_lower_than(1, 25, 0): assert config.description == "Test" else: assert config.description is None @@ -459,7 +461,9 @@ def test_collection_config_update(collection_factory: CollectionFactory) -> None ) config = collection.config.get() - if collection._connection._weaviate_version.is_at_least(1, 25, 2): + if collection._connection._weaviate_version.is_at_least( + 1, 25, 2 + ) or collection._connection._weaviate_version.is_lower_than(1, 25, 0): assert config.description == "Test" else: assert config.description is None