From 085e1b7cb9ad69179d1f8d9884a77bbcd99010e2 Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:41:41 +0200 Subject: [PATCH 1/3] update elasticsearch doc for `use_ssl` parameter --- docs/apache-airflow-providers-elasticsearch/logging/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-elasticsearch/logging/index.rst b/docs/apache-airflow-providers-elasticsearch/logging/index.rst index eb2acc55d224b..c1eac7154e278 100644 --- a/docs/apache-airflow-providers-elasticsearch/logging/index.rst +++ b/docs/apache-airflow-providers-elasticsearch/logging/index.rst @@ -61,7 +61,7 @@ cert, etc.) use the ``elasticsearch_configs`` setting in your ``airflow.cfg`` remote_logging = True [elasticsearch_configs] - use_ssl=True + use_ssl=True # (This setting is valid only for ``elasticsearch<8``) verify_certs=True ca_certs=/path/to/CA_certs From 77e32e4b1428eaea73ea113243c97e101d5fe27a Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Sat, 25 Nov 2023 21:12:53 +0200 Subject: [PATCH 2/3] Remove outdated support for elasticsearch<8 --- .../providers/elasticsearch/log/es_task_handler.py | 11 ----------- .../logging/index.rst | 1 - .../elasticsearch/log/test_es_task_handler.py | 7 +------ 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py b/airflow/providers/elasticsearch/log/es_task_handler.py index 79f9ad0b41994..f554444e346d9 100644 --- a/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/airflow/providers/elasticsearch/log/es_task_handler.py @@ -70,17 +70,6 @@ def get_es_kwargs_from_config() -> dict[str, Any]: if elastic_search_config else {} ) - # For elasticsearch>8 retry_timeout have changed for elasticsearch to retry_on_timeout - # in Elasticsearch() compared to previous versions. - # Read more at: https://elasticsearch-py.readthedocs.io/en/v8.8.2/api.html#module-elasticsearch - if ( - elastic_search_config - and "retry_timeout" in elastic_search_config - and not kwargs_dict.get("retry_on_timeout") - ): - retry_timeout = elastic_search_config.get("retry_timeout") - if retry_timeout is not None: - kwargs_dict["retry_on_timeout"] = retry_timeout return kwargs_dict diff --git a/docs/apache-airflow-providers-elasticsearch/logging/index.rst b/docs/apache-airflow-providers-elasticsearch/logging/index.rst index c1eac7154e278..fcb943a3d8734 100644 --- a/docs/apache-airflow-providers-elasticsearch/logging/index.rst +++ b/docs/apache-airflow-providers-elasticsearch/logging/index.rst @@ -61,7 +61,6 @@ cert, etc.) use the ``elasticsearch_configs`` setting in your ``airflow.cfg`` remote_logging = True [elasticsearch_configs] - use_ssl=True # (This setting is valid only for ``elasticsearch<8``) verify_certs=True ca_certs=/path/to/CA_certs diff --git a/tests/providers/elasticsearch/log/test_es_task_handler.py b/tests/providers/elasticsearch/log/test_es_task_handler.py index ec2bd3016a217..0eaf9975cd23f 100644 --- a/tests/providers/elasticsearch/log/test_es_task_handler.py +++ b/tests/providers/elasticsearch/log/test_es_task_handler.py @@ -671,14 +671,11 @@ def test_retrieve_config_keys(): """ with conf_vars( { - ("elasticsearch_configs", "use_ssl"): "True", ("elasticsearch_configs", "http_compress"): "False", ("elasticsearch_configs", "timeout"): "10", } ): args_from_config = get_es_kwargs_from_config().keys() - # use_ssl is removed from config - assert "use_ssl" not in args_from_config # verify_certs comes from default config value assert "verify_certs" in args_from_config # timeout comes from config provided value @@ -694,12 +691,10 @@ def test_retrieve_retry_on_timeout(): """ with conf_vars( { - ("elasticsearch_configs", "retry_timeout"): "True", + ("elasticsearch_configs", "retry_on_timeout"): "True", } ): args_from_config = get_es_kwargs_from_config().keys() - # use_ssl is removed from config - assert "retry_timeout" not in args_from_config # verify_certs comes from default config value assert "retry_on_timeout" in args_from_config From a7f015048f659c02ae116f89fcb77f797062e853 Mon Sep 17 00:00:00 2001 From: Elad Kalif <45845474+eladkal@users.noreply.github.com> Date: Wed, 29 Nov 2023 21:00:57 +0200 Subject: [PATCH 3/3] bring back retry_timeout --- .../providers/elasticsearch/log/es_task_handler.py | 14 ++++++++++++++ .../elasticsearch/log/test_es_task_handler.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py b/airflow/providers/elasticsearch/log/es_task_handler.py index f554444e346d9..28e4560dfbff6 100644 --- a/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/airflow/providers/elasticsearch/log/es_task_handler.py @@ -70,6 +70,20 @@ def get_es_kwargs_from_config() -> dict[str, Any]: if elastic_search_config else {} ) + # TODO: Remove in next major release (drop support for elasticsearch<8 parameters) + if ( + elastic_search_config + and "retry_timeout" in elastic_search_config + and not kwargs_dict.get("retry_on_timeout") + ): + warnings.warn( + "retry_timeout is not supported with elasticsearch>=8. Please use `retry_on_timeout`.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + retry_timeout = elastic_search_config.get("retry_timeout") + if retry_timeout is not None: + kwargs_dict["retry_on_timeout"] = retry_timeout return kwargs_dict diff --git a/tests/providers/elasticsearch/log/test_es_task_handler.py b/tests/providers/elasticsearch/log/test_es_task_handler.py index 0eaf9975cd23f..4a6e551b5d1b9 100644 --- a/tests/providers/elasticsearch/log/test_es_task_handler.py +++ b/tests/providers/elasticsearch/log/test_es_task_handler.py @@ -691,7 +691,7 @@ def test_retrieve_retry_on_timeout(): """ with conf_vars( { - ("elasticsearch_configs", "retry_on_timeout"): "True", + ("elasticsearch_configs", "retry_timeout"): "True", } ): args_from_config = get_es_kwargs_from_config().keys()