Check probes more frequently within the same amount of total time (#86)

* Check probes more frequently within same amount of total time Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Generate examples Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Remove initial readiness wait and incrase liveness for the components in statefulset Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>
thanos-io · Jan 17, 2020 · b6a6a41 · b6a6a41
1 parent 0c90793
commit b6a6a41
Show file tree

Hide file tree

Showing 14 changed files with 37 additions and 35 deletions.
diff --git a/examples/all/manifests/thanos-bucket-deployment.yaml b/examples/all/manifests/thanos-bucket-deployment.yaml
@@ -39,12 +39,12 @@ spec:
         - containerPort: 8080
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 8080
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: 250m

diff --git a/examples/all/manifests/thanos-compactor-statefulSet.yaml b/examples/all/manifests/thanos-compactor-statefulSet.yaml
@@ -44,12 +44,12 @@ spec:
         - containerPort: 10902
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 10902
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: 500m

diff --git a/examples/all/manifests/thanos-querier-deployment.yaml b/examples/all/manifests/thanos-querier-deployment.yaml
@@ -55,12 +55,12 @@ spec:
         - containerPort: 9090
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 9090
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "1"

diff --git a/examples/all/manifests/thanos-receive-statefulSet.yaml b/examples/all/manifests/thanos-receive-statefulSet.yaml
@@ -52,7 +52,7 @@ spec:
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
         livenessProbe:
-          failureThreshold: 4
+          failureThreshold: 8
           httpGet:
             path: /-/healthy
             port: 10902
@@ -67,12 +67,12 @@ spec:
         - containerPort: 19291
           name: remote-write
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 10902
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "1"

diff --git a/examples/all/manifests/thanos-ruler-statefulSet.yaml b/examples/all/manifests/thanos-ruler-statefulSet.yaml
@@ -38,25 +38,26 @@ spec:
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
         livenessProbe:
-          failureThreshold: 4
+          failureThreshold: 24
           httpGet:
             path: /-/healthy
             port: 10902
             scheme: HTTP
-          periodSeconds: 30
+          periodSeconds: 5
         name: thanos-ruler
         ports:
         - containerPort: 10901
           name: grpc
         - containerPort: 10902
           name: http
         readinessProbe:
+          failureThreshold: 18
           httpGet:
             path: /-/ready
             port: 10902
             scheme: HTTP
           initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "1"

diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml
@@ -31,7 +31,7 @@ spec:
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
         livenessProbe:
-          failureThreshold: 4
+          failureThreshold: 8
           httpGet:
             path: /-/healthy
             port: 10902
@@ -44,12 +44,12 @@ spec:
         - containerPort: 10902
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 10902
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "2"

diff --git a/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet b/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet
@@ -58,8 +58,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
-          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(20) +
           container.mixin.readinessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet b/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet
@@ -63,8 +63,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
-          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(20) +
           container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/jsonnet/kube-thanos/kube-thanos-querier.libsonnet b/jsonnet/kube-thanos/kube-thanos-querier.libsonnet
@@ -55,8 +55,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
-          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(20) +
           container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/jsonnet/kube-thanos/kube-thanos-receive.libsonnet b/jsonnet/kube-thanos/kube-thanos-receive.libsonnet
@@ -68,13 +68,13 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           ]) +
           container.mixin.livenessProbe +
           container.mixin.livenessProbe.withPeriodSeconds(30) +
-          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.withFailureThreshold(8) +
           container.mixin.livenessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
-          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(20) +
           container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet b/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet
@@ -82,14 +82,15 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
             { name: 'http', containerPort: tr.ports.http },
           ]) +
           container.mixin.livenessProbe +
-          container.mixin.livenessProbe.withPeriodSeconds(30) +
-          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.withPeriodSeconds(5) +
+          container.mixin.livenessProbe.withFailureThreshold(24) +
           container.mixin.livenessProbe.httpGet.withPort(tr.ports.http) +
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
           container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(18) +
           container.mixin.readinessProbe.httpGet.withPort(tr.ports.http) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet
@@ -66,13 +66,13 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           ]) +
           container.mixin.livenessProbe +
           container.mixin.livenessProbe.withPeriodSeconds(30) +
-          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.withFailureThreshold(8) +
           container.mixin.livenessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
           container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
           container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
           container.mixin.readinessProbe +
-          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
-          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.withPeriodSeconds(5) +
+          container.mixin.readinessProbe.withFailureThreshold(20) +
           container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
           container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
           container.mixin.readinessProbe.httpGet.withPath('/-/ready');

diff --git a/manifests/thanos-querier-deployment.yaml b/manifests/thanos-querier-deployment.yaml
@@ -51,12 +51,12 @@ spec:
         - containerPort: 9090
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 9090
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "1"

diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml
@@ -31,7 +31,7 @@ spec:
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
         livenessProbe:
-          failureThreshold: 4
+          failureThreshold: 8
           httpGet:
             path: /-/healthy
             port: 10902
@@ -44,12 +44,12 @@ spec:
         - containerPort: 10902
           name: http
         readinessProbe:
+          failureThreshold: 20
           httpGet:
             path: /-/ready
             port: 10902
             scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 30
+          periodSeconds: 5
         resources:
           limits:
             cpu: "2"