From 7af11b1b51346be45413ed428077a7b0fa8189f4 Mon Sep 17 00:00:00 2001 From: Gavin Williams Date: Wed, 20 Nov 2019 13:20:25 +0000 Subject: [PATCH] [elasticsearch] Tweak the 'readinessProbe' command to verify that master nodes are available. The behaviour of the `/` endpoint changed[0] between 6.x and 7.x, whereby previously it would return a HTTP `503` response when the cluster was blocked, it now returns a HTTP `200` response even if there are no masters available. This change updates the behaviour of the `readinessProbe` command during normal running to verify that the local node is responding and that there are master nodes available. [1] The desired behaviour here is that if the data nodes are unable to talk to their master nodes for whatever reason, then the data nodes will become `Unready` and therefore be removed from the Service load-balancer until the master nodes are available again. Refs: [0] https://github.com/elastic/elasticsearch/pull/29045 [1] https://github.com/elastic/elasticsearch/issues/34897#issuecomment-519441101 --- elasticsearch/templates/statefulset.yaml | 4 ++-- elasticsearch/tests/elasticsearch_test.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 593504fcf..7642daf65 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -211,8 +211,8 @@ spec: } if [ -f "${START_FILE}" ]; then - echo 'Elasticsearch is already running, lets check the node is healthy' - http "/" + echo 'Elasticsearch is already running, lets check the node is healthy and there are master nodes available' + http "/_cluster/health?timeout=0s" else echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' if http "/_cluster/health?{{ .Values.clusterHealthCheckParams }}" ; then diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 2e88b5e8d..978284aea 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -107,6 +107,7 @@ def test_defaults(): assert 'curl' in c['readinessProbe']['exec']['command'][-1] assert 'http://127.0.0.1:9200' in c['readinessProbe']['exec']['command'][-1] + assert '/_cluster/health?timeout=0s' in c['readinessProbe']['exec']['command'][-1] # Resources assert c['resources'] == {