From 860860cfbe69f3388c8109ba216ec1876d547446 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Fri, 30 Jun 2023 15:33:57 +0100 Subject: [PATCH] Disable index counter on `inga` to restore service in `prod` It seems locking contention in leveldb used by index counter causes high CPU spikes that then results in readiness check failure and ultimately cause all lookups to cid.contact to fail. To mitigate the issue and restore service, deploy a quick fix where index counter is completely excluded from execution path until appropriate fixes are in place. This will assure that the service is restored in the meantime. However, it means that we may end up with inaccurate index counts for some providers. This is an acceptable tradeoff since absolute index counts are not as important than lookup service itself or their relative growth. --- .../tenant/storetheindex/indexstar/deployment.yaml | 8 ++++---- .../storetheindex/instances/inga/kustomization.yaml | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/indexstar/deployment.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/indexstar/deployment.yaml index 7f9999116..d4f44fd14 100644 --- a/deploy/manifests/prod/us-east-2/tenant/storetheindex/indexstar/deployment.yaml +++ b/deploy/manifests/prod/us-east-2/tenant/storetheindex/indexstar/deployment.yaml @@ -22,10 +22,10 @@ spec: # - '--providersBackends=http://cera-indexer:3000/' # Remove old nodes since inga is largely caught up and other indexers may be further behind/forward. - # This results in inconsistent responses and hard to debugt issues. - - '--providersBackends=http://oden-indexer:3000/' - - '--providersBackends=http://kepa-indexer:3000/' - - '--providersBackends=http://dido-indexer:3000/' + # This results in inconsistent responses and hard to debug issues. + # - '--providersBackends=http://oden-indexer:3000/' + # - '--providersBackends=http://kepa-indexer:3000/' + # - '--providersBackends=http://dido-indexer:3000/' - '--backends=http://dhfind.internal.prod.cid.contact/' - '--backends=http://dhfind-helga.internal.prod.cid.contact/' - '--backends=http://dhfind-porvy.internal.prod.cid.contact/' diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/kustomization.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/kustomization.yaml index 579a7c45a..a5122fcc1 100644 --- a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/kustomization.yaml +++ b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/kustomization.yaml @@ -27,3 +27,12 @@ configMapGenerator: patchesStrategicMerge: - deployment.yaml + +images: + - name: storetheindex + newName: 407967248065.dkr.ecr.us-east-2.amazonaws.com/storetheindex/storetheindex + # Deploy a branch build that disables index counter from both ingest and find server + # in order to relieve high cpu spikes that cause lookup outage. See: + # - https://github.com/ipni/storetheindex/issues/2062 + # - https://github.com/ipni/storetheindex/pull/2067 + newTag: 20230630135351-bc7040fa3bcceac092753d69a0a2040c683351b3