From c0a48718597fb41f131eca0f9993441241d25480 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:46:41 +0000 Subject: [PATCH] support composable templates (#1943) (#1952) * support composable templates * do not error on aiohttp warning * rename new index template to composable index template * remove old comment (cherry picked from commit a0c53db06161a347c06fae61c2ffa7563fa0e2c1) Co-authored-by: Miguel Grinberg --- .gitignore | 2 + elasticsearch_dsl/__init__.py | 11 +++- elasticsearch_dsl/_async/index.py | 54 +++++++++++++++++-- elasticsearch_dsl/_sync/index.py | 50 +++++++++++++++-- elasticsearch_dsl/index.py | 8 ++- examples/alias_migration.py | 5 +- examples/async/alias_migration.py | 5 +- examples/async/parent_child.py | 2 +- examples/parent_child.py | 2 +- setup.cfg | 1 + tests/conftest.py | 4 ++ tests/test_integration/_async/test_index.py | 27 +++++++++- tests/test_integration/_sync/test_index.py | 36 ++++++++++++- .../_async/test_alias_migration.py | 2 +- .../test_examples/_async/test_parent_child.py | 2 +- .../_sync/test_alias_migration.py | 2 +- .../test_examples/_sync/test_parent_child.py | 2 +- utils/run-unasync.py | 1 + 18 files changed, 196 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index ebf415e96..d857636f5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ venv # sample code for GitHub issues issues +.direnv +.envrc diff --git a/elasticsearch_dsl/__init__.py b/elasticsearch_dsl/__init__.py index 31ba148f7..55ca065e6 100644 --- a/elasticsearch_dsl/__init__.py +++ b/elasticsearch_dsl/__init__.py @@ -79,7 +79,14 @@ construct_field, ) from .function import SF -from .index import AsyncIndex, AsyncIndexTemplate, Index, IndexTemplate +from .index import ( + AsyncComposableIndexTemplate, + AsyncIndex, + AsyncIndexTemplate, + ComposableIndexTemplate, + Index, + IndexTemplate, +) from .mapping import AsyncMapping, Mapping from .query import Q, Query from .response import AggResponse, Response, UpdateByQueryResponse @@ -102,6 +109,7 @@ "A", "Agg", "AggResponse", + "AsyncComposableIndexTemplate", "AsyncDocument", "AsyncEmptySearch", "AsyncFacetedSearch", @@ -117,6 +125,7 @@ "Boolean", "Byte", "Completion", + "ComposableIndexTemplate", "ConstantKeyword", "CustomField", "Date", diff --git a/elasticsearch_dsl/_async/index.py b/elasticsearch_dsl/_async/index.py index 765e74385..22da1b14e 100644 --- a/elasticsearch_dsl/_async/index.py +++ b/elasticsearch_dsl/_async/index.py @@ -73,6 +73,47 @@ async def save( ) +class AsyncComposableIndexTemplate: + def __init__( + self, + name: str, + template: str, + index: Optional["AsyncIndex"] = None, + priority: Optional[int] = None, + **kwargs: Any, + ): + if index is None: + self._index = AsyncIndex(template, **kwargs) + else: + if kwargs: + raise ValueError( + "You cannot specify options for Index when" + " passing an Index instance." + ) + self._index = index.clone() + self._index._name = template + self._template_name = name + self.priority = priority + + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._index, attr_name) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"template": self._index.to_dict()} + d["index_patterns"] = [self._index._name] + if self.priority is not None: + d["priority"] = self.priority + return d + + async def save( + self, using: Optional[AsyncUsingType] = None + ) -> "ObjectApiResponse[Any]": + es = get_connection(using or self._index._using) + return await es.indices.put_index_template( + name=self._template_name, **self.to_dict() + ) + + class AsyncIndex(IndexBase): _using: AsyncUsingType @@ -102,13 +143,20 @@ def as_template( pattern: Optional[str] = None, order: Optional[int] = None, ) -> AsyncIndexTemplate: - # TODO: should we allow pattern to be a top-level arg? - # or maybe have an IndexPattern that allows for it and have - # Document._index be that? return AsyncIndexTemplate( template_name, pattern or self._name, index=self, order=order ) + def as_composable_template( + self, + template_name: str, + pattern: Optional[str] = None, + priority: Optional[int] = None, + ) -> AsyncComposableIndexTemplate: + return AsyncComposableIndexTemplate( + template_name, pattern or self._name, index=self, priority=priority + ) + async def load_mappings(self, using: Optional[AsyncUsingType] = None) -> None: await self.get_or_create_mapping().update_from_es( self._name, using=using or self._using diff --git a/elasticsearch_dsl/_sync/index.py b/elasticsearch_dsl/_sync/index.py index 59508d513..3b562a674 100644 --- a/elasticsearch_dsl/_sync/index.py +++ b/elasticsearch_dsl/_sync/index.py @@ -69,6 +69,43 @@ def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]": return es.indices.put_template(name=self._template_name, body=self.to_dict()) +class ComposableIndexTemplate: + def __init__( + self, + name: str, + template: str, + index: Optional["Index"] = None, + priority: Optional[int] = None, + **kwargs: Any, + ): + if index is None: + self._index = Index(template, **kwargs) + else: + if kwargs: + raise ValueError( + "You cannot specify options for Index when" + " passing an Index instance." + ) + self._index = index.clone() + self._index._name = template + self._template_name = name + self.priority = priority + + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._index, attr_name) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"template": self._index.to_dict()} + d["index_patterns"] = [self._index._name] + if self.priority is not None: + d["priority"] = self.priority + return d + + def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]": + es = get_connection(using or self._index._using) + return es.indices.put_index_template(name=self._template_name, **self.to_dict()) + + class Index(IndexBase): _using: UsingType @@ -96,13 +133,20 @@ def as_template( pattern: Optional[str] = None, order: Optional[int] = None, ) -> IndexTemplate: - # TODO: should we allow pattern to be a top-level arg? - # or maybe have an IndexPattern that allows for it and have - # Document._index be that? return IndexTemplate( template_name, pattern or self._name, index=self, order=order ) + def as_composable_template( + self, + template_name: str, + pattern: Optional[str] = None, + priority: Optional[int] = None, + ) -> ComposableIndexTemplate: + return ComposableIndexTemplate( + template_name, pattern or self._name, index=self, priority=priority + ) + def load_mappings(self, using: Optional[UsingType] = None) -> None: self.get_or_create_mapping().update_from_es( self._name, using=using or self._using diff --git a/elasticsearch_dsl/index.py b/elasticsearch_dsl/index.py index ef9f4b0b9..368e58d42 100644 --- a/elasticsearch_dsl/index.py +++ b/elasticsearch_dsl/index.py @@ -15,5 +15,9 @@ # specific language governing permissions and limitations # under the License. -from ._async.index import AsyncIndex, AsyncIndexTemplate # noqa: F401 -from ._sync.index import Index, IndexTemplate # noqa: F401 +from ._async.index import ( # noqa: F401 + AsyncComposableIndexTemplate, + AsyncIndex, + AsyncIndexTemplate, +) +from ._sync.index import ComposableIndexTemplate, Index, IndexTemplate # noqa: F401 diff --git a/examples/alias_migration.py b/examples/alias_migration.py index c9fe4ede5..ba1204a99 100644 --- a/examples/alias_migration.py +++ b/examples/alias_migration.py @@ -44,6 +44,7 @@ ALIAS = "test-blog" PATTERN = ALIAS + "-*" +PRIORITY = 100 class BlogPost(Document): @@ -81,7 +82,9 @@ def setup() -> None: deploy. """ # create an index template - index_template = BlogPost._index.as_template(ALIAS, PATTERN) + index_template = BlogPost._index.as_composable_template( + ALIAS, PATTERN, priority=PRIORITY + ) # upload the template into elasticsearch # potentially overriding the one already there index_template.save() diff --git a/examples/async/alias_migration.py b/examples/async/alias_migration.py index bede90980..2f9dbf736 100644 --- a/examples/async/alias_migration.py +++ b/examples/async/alias_migration.py @@ -45,6 +45,7 @@ ALIAS = "test-blog" PATTERN = ALIAS + "-*" +PRIORITY = 100 class BlogPost(AsyncDocument): @@ -82,7 +83,9 @@ async def setup() -> None: deploy. """ # create an index template - index_template = BlogPost._index.as_template(ALIAS, PATTERN) + index_template = BlogPost._index.as_composable_template( + ALIAS, PATTERN, priority=PRIORITY + ) # upload the template into elasticsearch # potentially overriding the one already there await index_template.save() diff --git a/examples/async/parent_child.py b/examples/async/parent_child.py index 6668a77c4..c96455eda 100644 --- a/examples/async/parent_child.py +++ b/examples/async/parent_child.py @@ -226,7 +226,7 @@ async def save(self, **kwargs: Any) -> None: # type: ignore[override] async def setup() -> None: """Create an IndexTemplate and save it into elasticsearch.""" - index_template = Post._index.as_template("base") + index_template = Post._index.as_composable_template("base", priority=100) await index_template.save() diff --git a/examples/parent_child.py b/examples/parent_child.py index 5acbbd727..09b03adb2 100644 --- a/examples/parent_child.py +++ b/examples/parent_child.py @@ -225,7 +225,7 @@ def save(self, **kwargs: Any) -> None: # type: ignore[override] def setup() -> None: """Create an IndexTemplate and save it into elasticsearch.""" - index_template = Post._index.as_template("base") + index_template = Post._index.as_composable_template("base", priority=100) index_template.save() diff --git a/setup.cfg b/setup.cfg index fc099da63..dab811f90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,5 +11,6 @@ filterwarnings = error ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning + default:enable_cleanup_closed ignored.*:DeprecationWarning markers = sync: mark a test as performing I/O without asyncio. diff --git a/tests/conftest.py b/tests/conftest.py index 1a07670e9..6a2589b9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,6 +122,7 @@ def teardown_method(self, _: Any) -> None: ) self.client.indices.delete(index="*", expand_wildcards=expand_wildcards) self.client.indices.delete_template(name="*") + self.client.indices.delete_index_template(name="*") def es_version(self) -> Tuple[int, ...]: if not hasattr(self, "_es_version"): @@ -172,6 +173,9 @@ def write_client(client: Elasticsearch) -> Generator[Elasticsearch, None, None]: for index_name in client.indices.get(index="test-*", expand_wildcards="all"): client.indices.delete(index=index_name) client.options(ignore_status=404).indices.delete_template(name="test-template") + client.options(ignore_status=404).indices.delete_index_template( + name="test-template" + ) @pytest_asyncio.fixture diff --git a/tests/test_integration/_async/test_index.py b/tests/test_integration/_async/test_index.py index 21e4fa7cd..db570cc73 100644 --- a/tests/test_integration/_async/test_index.py +++ b/tests/test_integration/_async/test_index.py @@ -19,6 +19,7 @@ from elasticsearch import AsyncElasticsearch from elasticsearch_dsl import ( + AsyncComposableIndexTemplate, AsyncDocument, AsyncIndex, AsyncIndexTemplate, @@ -35,7 +36,31 @@ class Post(AsyncDocument): @pytest.mark.asyncio async def test_index_template_works(async_write_client: AsyncElasticsearch) -> None: - it = AsyncIndexTemplate("test-template", "test-*") + it = AsyncIndexTemplate("test-template", "test-legacy-*") + it.document(Post) + it.settings(number_of_replicas=0, number_of_shards=1) + await it.save() + + i = AsyncIndex("test-legacy-blog") + await i.create() + + assert { + "test-legacy-blog": { + "mappings": { + "properties": { + "title": {"type": "text", "analyzer": "my_analyzer"}, + "published_from": {"type": "date"}, + } + } + } + } == await async_write_client.indices.get_mapping(index="test-legacy-blog") + + +@pytest.mark.asyncio +async def test_composable_index_template_works( + async_write_client: AsyncElasticsearch, +) -> None: + it = AsyncComposableIndexTemplate("test-template", "test-*") it.document(Post) it.settings(number_of_replicas=0, number_of_shards=1) await it.save() diff --git a/tests/test_integration/_sync/test_index.py b/tests/test_integration/_sync/test_index.py index ff435bdfb..9f8da73ee 100644 --- a/tests/test_integration/_sync/test_index.py +++ b/tests/test_integration/_sync/test_index.py @@ -18,7 +18,15 @@ import pytest from elasticsearch import Elasticsearch -from elasticsearch_dsl import Date, Document, Index, IndexTemplate, Text, analysis +from elasticsearch_dsl import ( + ComposableIndexTemplate, + Date, + Document, + Index, + IndexTemplate, + Text, + analysis, +) class Post(Document): @@ -28,7 +36,31 @@ class Post(Document): @pytest.mark.sync def test_index_template_works(write_client: Elasticsearch) -> None: - it = IndexTemplate("test-template", "test-*") + it = IndexTemplate("test-template", "test-legacy-*") + it.document(Post) + it.settings(number_of_replicas=0, number_of_shards=1) + it.save() + + i = Index("test-legacy-blog") + i.create() + + assert { + "test-legacy-blog": { + "mappings": { + "properties": { + "title": {"type": "text", "analyzer": "my_analyzer"}, + "published_from": {"type": "date"}, + } + } + } + } == write_client.indices.get_mapping(index="test-legacy-blog") + + +@pytest.mark.sync +def test_composable_index_template_works( + write_client: Elasticsearch, +) -> None: + it = ComposableIndexTemplate("test-template", "test-*") it.document(Post) it.settings(number_of_replicas=0, number_of_shards=1) it.save() diff --git a/tests/test_integration/test_examples/_async/test_alias_migration.py b/tests/test_integration/test_examples/_async/test_alias_migration.py index 81202706b..dae4c973f 100644 --- a/tests/test_integration/test_examples/_async/test_alias_migration.py +++ b/tests/test_integration/test_examples/_async/test_alias_migration.py @@ -28,7 +28,7 @@ async def test_alias_migration(async_write_client: AsyncElasticsearch) -> None: await alias_migration.setup() # verify that template, index, and alias has been set up - assert await async_write_client.indices.exists_template(name=ALIAS) + assert await async_write_client.indices.exists_index_template(name=ALIAS) assert await async_write_client.indices.exists(index=PATTERN) assert await async_write_client.indices.exists_alias(name=ALIAS) diff --git a/tests/test_integration/test_examples/_async/test_parent_child.py b/tests/test_integration/test_examples/_async/test_parent_child.py index 9a1027f4f..b12d445fd 100644 --- a/tests/test_integration/test_examples/_async/test_parent_child.py +++ b/tests/test_integration/test_examples/_async/test_parent_child.py @@ -45,7 +45,7 @@ @pytest_asyncio.fixture async def question(async_write_client: AsyncElasticsearch) -> Question: await setup() - assert await async_write_client.indices.exists_template(name="base") + assert await async_write_client.indices.exists_index_template(name="base") # create a question object q = Question( diff --git a/tests/test_integration/test_examples/_sync/test_alias_migration.py b/tests/test_integration/test_examples/_sync/test_alias_migration.py index 59cdb372c..9a74b699b 100644 --- a/tests/test_integration/test_examples/_sync/test_alias_migration.py +++ b/tests/test_integration/test_examples/_sync/test_alias_migration.py @@ -28,7 +28,7 @@ def test_alias_migration(write_client: Elasticsearch) -> None: alias_migration.setup() # verify that template, index, and alias has been set up - assert write_client.indices.exists_template(name=ALIAS) + assert write_client.indices.exists_index_template(name=ALIAS) assert write_client.indices.exists(index=PATTERN) assert write_client.indices.exists_alias(name=ALIAS) diff --git a/tests/test_integration/test_examples/_sync/test_parent_child.py b/tests/test_integration/test_examples/_sync/test_parent_child.py index dcbbde867..12d93914d 100644 --- a/tests/test_integration/test_examples/_sync/test_parent_child.py +++ b/tests/test_integration/test_examples/_sync/test_parent_child.py @@ -44,7 +44,7 @@ @pytest.fixture def question(write_client: Elasticsearch) -> Question: setup() - assert write_client.indices.exists_template(name="base") + assert write_client.indices.exists_index_template(name="base") # create a question object q = Question( diff --git a/utils/run-unasync.py b/utils/run-unasync.py index bae0c7a6f..7a55e7b59 100644 --- a/utils/run-unasync.py +++ b/utils/run-unasync.py @@ -57,6 +57,7 @@ def main(check=False): "AsyncIndexMeta": "IndexMeta", "AsyncIndexTemplate": "IndexTemplate", "AsyncIndex": "Index", + "AsyncComposableIndexTemplate": "ComposableIndexTemplate", "AsyncUpdateByQuery": "UpdateByQuery", "AsyncMapping": "Mapping", "AsyncFacetedSearch": "FacetedSearch",