New indexers for indexing with FDB (#1906)

* New indexers for indexing with FDB * Use r6a.2xlarge instances for new indexers * Reduce instance CPU requests to 7 This would leave 1 CPU for daemon sets that would be scheduled on the machine to collect logs etc, since the instance type offers only 8 CPUs.
ipni · Jun 8, 2023 · a9b3463 · a9b3463
1 parent 0f2ad51
commit a9b3463
Show file tree

Hide file tree

Showing 19 changed files with 723 additions and 0 deletions.
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/config.json b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/config.json
@@ -0,0 +1,105 @@
+{
+  "Version": 2,
+  "Identity": {
+    "PeerID": "",
+    "PrivKey": ""
+  },
+  "Addresses": {
+    "Admin": "/ip4/0.0.0.0/tcp/3002",
+    "Finder": "/ip4/0.0.0.0/tcp/3000",
+    "FinderWebpage": "https://web-ipni.cid.contact/",
+    "Ingest": "/ip4/0.0.0.0/tcp/3001",
+    "P2PAddr": "/ip4/0.0.0.0/tcp/3003",
+    "NoResourceManager": true
+  },
+  "Bootstrap": {
+    "Peers": [
+      "/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d",
+      "/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R",
+      "/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc",
+      "/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ",
+      "/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf",
+      "/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR",
+      "/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt",
+      "/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4",
+      "/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH",
+      "/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u",
+      "/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj",
+      "/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc",
+      "/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz",
+      "/dns4/lotus-bootstrap.ipfsforce.com/tcp/41778/p2p/12D3KooWGhufNmZHF3sv48aQeS13ng5XVJZ9E6qy2Ms4VzqeUsHk",
+      "/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP"
+    ],
+    "MinimumPeers": 4
+  },
+  "Datastore": {
+    "Dir": "/data/datastore",
+    "Type": "levelds",
+    "TmpDir": "/data/tmpstore",
+    "TmpType": "levelds"
+  },
+  "Discovery": {
+    "FilterIPs": true,
+    "Policy": {
+      "Allow": true,
+      "Except": null,
+      "Publish": true,
+      "PublishExcept": null
+    },
+    "PollInterval": "24h0m0s",
+    "PollRetryAfter": "5m0s",
+    "PollStopAfter": "168h0m0s",
+    "PollOverrides": null,
+    "UseAssigner": true
+  },
+  "Indexer": {
+    "CacheSize": -1,
+    "ConfigCheckInterval": "30s",
+    "ShutdownTimeout": "15m",
+    "ValueStoreDir": "",
+    "ValueStoreType": "none",
+    "DHBatchSize": 16384,
+    "DHStoreURL": "http://dhstore-stateless:40080",
+    "DHStoreHttpClientTimeout": "30s"
+  },
+  "Ingest": {
+    "AdvertisementDepthLimit": 33554432,
+    "AdvertisementMirror": {
+      "Read": true,
+      "Write": true,
+      "Compress": "gzip",
+      "Storage": {
+        "Type": "s3",
+        "S3": {
+          "BucketName": "prod-sti-adstore"
+        }
+      }
+    },
+    "EntriesDepthLimit": 65536,
+    "HttpSyncRetryMax": 4,
+    "HttpSyncRetryWaitMax": "30s",
+    "HttpSyncRetryWaitMin": "1s",
+    "HttpSyncTimeout": "10s",
+    "IngestWorkerCount":15,
+    "PubSubTopic": "/indexer/ingest/mainnet",
+    "ResendDirectAnnounce": false,
+    "StoreBatchSize": 8192,
+    "SyncSegmentDepthLimit": 2000,
+    "SyncTimeout": "2h0m0s"
+  },
+  "Logging": {
+    "Level": "info",
+    "Loggers": {
+      "basichost": "warn",
+      "bootstrap": "warn",
+      "dt-impl": "warn",
+      "dt_graphsync": "warn",
+      "graphsync": "warn"
+    }
+  },
+  "Peering": {
+    "Peers": [
+        "/dns4/assigner/tcp/3003/p2p/12D3KooWQAymjDKMivbkUNiJP7ChRsvsDuazerHW4wERRvQMWNor"
+    ]
+  }
+}
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/deployment.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/deployment.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: indexer
+spec:
+  template:
+    spec:
+      serviceAccountName: storetheindex
+      terminationGracePeriodSeconds: 600
+      containers:
+        - name: indexer
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            limits:
+              cpu: "7"
+              memory: 60Gi
+            requests:
+              cpu: "7"
+              memory: 60Gi
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node.kubernetes.io/instance-type
+                    operator: In
+                    values:
+                      - r6a.2xlarge
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - us-east-2a
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: arya-data
+
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/identity.key.encrypted b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/identity.key.encrypted
@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:bZGLMf9f/sF4d3XlAxpqkWtqskg6QmB0JQUdIC+lRvuM1W//CZE8tK3evJqrZGiJp+PnDSBCA4hyWlNpt/dfcpzLrUA=,iv:gaibZp0Vb+iwHTJl4hQOrydYAzNuNPaTt403vIpuw3A=,tag:LBu8jq65avfQWnUlt0Vw9g==,type:str]",
+	"sops": {
+		"kms": [
+			{
+				"arn": "arn:aws:kms:us-east-2:407967248065:alias/prod/us-east-2/sti",
+				"created_at": "2023-06-07T21:39:13Z",
+				"enc": "AQICAHjmLCaDZ4fRYyty7669VvFjJmy9C7/Y4dwd6seUJHRobwHSO1aJMk5mdMVR8Y0k9zhpAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMe3h5Xy0K0rBLvLvSAgEQgDvR+RQ/AcrRgWq3gWdPA9NCC3ldchUieza+eBcVPFho1f1li4KzqIH1NLHPdWISihP7RlseANmnUFjqaA==",
+				"aws_profile": ""
+			}
+		],
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2023-06-07T21:39:14Z",
+		"mac": "ENC[AES256_GCM,data:6mcUSiaHyP5g/VimAEB4+UmPy2oxS22rSSbwK0ol5il6pxHgQhhzYyzcLEKCLwPC+5CG3TW/tLpmx+jGsovfBfxO+EGzRgGvFm2IHj2zGSgiZXKtUT9ZvRAazf1hDASt5NFsqJVQWHjHindQBwH0zwxapB5RvHonKC7r7GaJn2s=,iv:pljKNcvlWW2ELttlUdCId/rvdMX0GUSXhG0NthJD/Ow=,tag:rGTBNthPNSC9frPydwFEyw==,type:str]",
+		"pgp": null,
+		"encrypted_regex": "^(data|stringData)$",
+		"version": "3.7.3"
+	}
+}
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/ingress.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/ingress.yaml
@@ -0,0 +1,30 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: indexer
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    cert-manager.io/cluster-issuer: "letsencrypt"
+spec:
+  tls:
+    - hosts:
+        - arya.prod.cid.contact
+      secretName: arya-indexer-ingress-tls
+  rules:
+    - host: arya.prod.cid.contact
+      http:
+        paths:
+          - path: /ingest
+            pathType: Prefix
+            backend:
+              service:
+                name: indexer
+                port:
+                  number: 3001
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: indexer
+                port:
+                  number: 3000
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/kustomization.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/kustomization.yaml
@@ -0,0 +1,29 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: storetheindex
+
+resources:
+  - ../../../../../../base/storetheindex-single
+  - ingress.yaml
+  - pvc_data.yaml
+
+namePrefix: arya-
+
+commonLabels:
+  name: arya
+
+secretGenerator:
+  - name: identity
+    behavior: replace
+    files:
+      - identity.key=identity.key.encrypted # 12D3KooWQoE7zwUzodJsLVSukj1tMtygRmNPBE6YZovcqkBFTQQF
+
+configMapGenerator:
+  - name: config
+    behavior: replace
+    files:
+      - config=config.json
+
+patchesStrategicMerge:
+  - deployment.yaml
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/pvc_data.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/arya/pvc_data.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app: indexer-single
+    app.kubernetes.io/managed-by: kustomization
+    app.kubernetes.io/part-of: storetheindex
+  name: data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Gi
+  storageClassName: gp3
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/bala/config.json b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/bala/config.json
@@ -0,0 +1,105 @@
+{
+  "Version": 2,
+  "Identity": {
+    "PeerID": "",
+    "PrivKey": ""
+  },
+  "Addresses": {
+    "Admin": "/ip4/0.0.0.0/tcp/3002",
+    "Finder": "/ip4/0.0.0.0/tcp/3000",
+    "FinderWebpage": "https://web-ipni.cid.contact/",
+    "Ingest": "/ip4/0.0.0.0/tcp/3001",
+    "P2PAddr": "/ip4/0.0.0.0/tcp/3003",
+    "NoResourceManager": true
+  },
+  "Bootstrap": {
+    "Peers": [
+      "/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d",
+      "/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R",
+      "/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc",
+      "/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ",
+      "/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf",
+      "/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR",
+      "/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt",
+      "/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4",
+      "/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH",
+      "/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u",
+      "/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj",
+      "/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc",
+      "/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz",
+      "/dns4/lotus-bootstrap.ipfsforce.com/tcp/41778/p2p/12D3KooWGhufNmZHF3sv48aQeS13ng5XVJZ9E6qy2Ms4VzqeUsHk",
+      "/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP"
+    ],
+    "MinimumPeers": 4
+  },
+  "Datastore": {
+    "Dir": "/data/datastore",
+    "Type": "levelds",
+    "TmpDir": "/data/tmpstore",
+    "TmpType": "levelds"
+  },
+  "Discovery": {
+    "FilterIPs": true,
+    "Policy": {
+      "Allow": true,
+      "Except": null,
+      "Publish": true,
+      "PublishExcept": null
+    },
+    "PollInterval": "24h0m0s",
+    "PollRetryAfter": "5m0s",
+    "PollStopAfter": "168h0m0s",
+    "PollOverrides": null,
+    "UseAssigner": true
+  },
+  "Indexer": {
+    "CacheSize": -1,
+    "ConfigCheckInterval": "30s",
+    "ShutdownTimeout": "15m",
+    "ValueStoreDir": "",
+    "ValueStoreType": "none",
+    "DHBatchSize": 16384,
+    "DHStoreURL": "http://dhstore-stateless:40080",
+    "DHStoreHttpClientTimeout": "30s"
+  },
+  "Ingest": {
+    "AdvertisementDepthLimit": 33554432,
+    "AdvertisementMirror": {
+      "Read": true,
+      "Write": true,
+      "Compress": "gzip",
+      "Storage": {
+        "Type": "s3",
+        "S3": {
+          "BucketName": "prod-sti-adstore"
+        }
+      }
+    },
+    "EntriesDepthLimit": 65536,
+    "HttpSyncRetryMax": 4,
+    "HttpSyncRetryWaitMax": "30s",
+    "HttpSyncRetryWaitMin": "1s",
+    "HttpSyncTimeout": "10s",
+    "IngestWorkerCount":15,
+    "PubSubTopic": "/indexer/ingest/mainnet",
+    "ResendDirectAnnounce": false,
+    "StoreBatchSize": 8192,
+    "SyncSegmentDepthLimit": 2000,
+    "SyncTimeout": "2h0m0s"
+  },
+  "Logging": {
+    "Level": "info",
+    "Loggers": {
+      "basichost": "warn",
+      "bootstrap": "warn",
+      "dt-impl": "warn",
+      "dt_graphsync": "warn",
+      "graphsync": "warn"
+    }
+  },
+  "Peering": {
+    "Peers": [
+        "/dns4/assigner/tcp/3003/p2p/12D3KooWQAymjDKMivbkUNiJP7ChRsvsDuazerHW4wERRvQMWNor"
+    ]
+  }
+}
diff --git a/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/bala/deployment.yaml b/deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/bala/deployment.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: indexer
+spec:
+  template:
+    spec:
+      serviceAccountName: storetheindex
+      terminationGracePeriodSeconds: 600
+      containers:
+        - name: indexer
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            limits:
+              cpu: "7"
+              memory: 60Gi
+            requests:
+              cpu: "7"
+              memory: 60Gi
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node.kubernetes.io/instance-type
+                    operator: In
+                    values:
+                      - r6a.2xlarge
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - us-east-2b
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: bala-data
+