Merge branch 'develop' of https://github.com/desci-labs/nodes into re…

…po-config
desci-labs · Oct 11, 2024 · f03e9a4 · f03e9a4
2 parents 8ec98bd + 77f3f8f
commit f03e9a4
Show file tree

Hide file tree

Showing 91 changed files with 10,661 additions and 583 deletions.
diff --git a/.ceramicDev.config.json b/.ceramicDev.config.json
@@ -1,5 +1,5 @@
 {
-  "anchor": { },
+  "anchor": {},
   "http-api": {
     "cors-allowed-origins": [
       ".*"
@@ -16,8 +16,8 @@
     "log-level": 2
   },
   "metrics": {
-    "metrics-exporter-enabled": false,
-    "metrics-port": 9090
+    "prometheus-exporter-enabled": true,
+    "prometheus-exporter-port": 9464
   },
   "network": {
     "name": "inmemory"
@@ -33,3 +33,4 @@
     "models": []
   }
 }
+
diff --git a/.env.example b/.env.example
@@ -119,7 +119,10 @@ ETHEREUM_RPC_URL=http://host.docker.internal:8545
 # Use this for Sepolia testnet
 # ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo
 
-DPID_URL_OVERRIDE=https://dev-beta.dpid.org
+# Use resolver in compose cluster locally
+DPID_URL_OVERRIDE=http://host.docker.internal:5460
+# Use public dev resolver
+# DPID_URL_OVERRIDE=https://dev-beta.dpid.org
 
 # Set to true if you want to mute the publish worker in local dev
 MUTE_PUBLISH_WORKER=false
@@ -139,8 +142,6 @@ CROSSREF_PASSWORD=
 # Cross ref notification callback envs
 CROSSREF_NOTIFY_ENDPOINT=endpoint
 
-# automated metadata
-
 # Automated metadata
 AUTOMATED_METADATA_API=http://host.docker.internal:5005
 AUTOMATED_METADATA_API_KEY=
@@ -158,6 +159,5 @@ ES_DB_NAME=
 ES_DB_USER=
 ES_DB_PASSWORD=
 
-
 ### open Alex Database - Postgres 
 OPEN_ALEX_DATABASE_URL=postgresql://username:password@host/database?schema=openalex
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -22,6 +22,7 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   CONTAINER_IMAGE: desci-server
   DOCKER_BUILDKIT: 1
+  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
 
 jobs:
   build-and-test:
@@ -55,6 +56,15 @@ jobs:
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
 
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+          cd desci-server && yarn build
+
       - name: Run tests
         run: |
           cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test

diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml
@@ -23,6 +23,7 @@ env:
   AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
   CONTAINER_IMAGE: desci-server
   DOCKER_BUILDKIT: 1
 
@@ -57,6 +58,14 @@ jobs:
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
 
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+
       - name: Run tests
         run: |
           cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
@@ -113,6 +122,14 @@ jobs:
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
 
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'
         run: |

diff --git a/ceramic-k8s/ceramic_deployment_dev.yaml b/ceramic-k8s/ceramic_deployment_dev.yaml
@@ -11,10 +11,7 @@ spec:
     matchLabels:
       App: JsCeramicDev
   strategy:
-    rollingUpdate:
-      maxSurge: 25%
-      maxUnavailable: 25%
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
@@ -32,7 +29,7 @@ spec:
 
           {{- end -}}
           {{- with secret "secrets/desci-server/dev/db" -}}
-          echo "{\"anchor\":{\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-dev-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"metrics-exporter-enabled\":false,\"metrics-port\":9090},\"network\":{\"name\":\"testnet-clay\"},\"node\":{},\"state-store\":{\"mode\":\"s3\",\"s3-bucket\":\"ceramic-node-dev-prime2\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"models\":[]}}" > daemon.config.json
+          echo "{\"anchor\":{\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-dev-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"prometheus-exporter-enabled\":true,\"prometheus-exporter-port\":9464},\"network\":{\"name\":\"testnet-clay\"},\"node\":{},\"state-store\":{\"mode\":\"s3\",\"s3-bucket\":\"ceramic-node-dev-prime2\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true}}" > daemon.config.json
           export NODE_ENV=production
           export CERAMIC_PUBSUB_QPS_LIMIT=500
           echo "envset"; 
@@ -41,20 +38,24 @@ spec:
         App: JsCeramicDev
     spec:
       containers:
-        - image: ceramicnetwork/js-ceramic:5.16.0
+          # Enables a fix allowing us to heal dropped anchors on genesis commits:
+          # https://github.com/ceramicnetwork/js-ceramic/pull/3280
+        - image: ceramicnetwork/js-ceramic:6.6.0-rc.0
           name: js-ceramic-dev
           command: ["/bin/bash", "-c"]
           args:
             - echo "SOURCING ENV"; source /vault/secrets/config; ./packages/cli/bin/ceramic.js daemon --config daemon.config.json;
           ports:
             - containerPort: 7007
               name: http-api
+            - containerPort: 9464
+              name: metrics
           resources:
             limits:
-              cpu: "0.7"
+              cpu: 2
               memory: 4Gi
             requests:
-              cpu: "0.6"
+              cpu: 1
               memory: 4Gi
           # # restart pod after failureThreshold*periodSeconds total seconds
           livenessProbe:
@@ -70,9 +71,38 @@ spec:
               port: http-api
             failureThreshold: 60
             periodSeconds: 1
-
       serviceAccountName: "vault-auth"
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: js-ceramic-dev-service-internal
+  labels:
+    App: JsCeramicDev
+spec:
+  type: ClusterIP
+  selector:
+    App: JsCeramicDev
+  ports:
+    - name: metrics
+      port: 9464
+      targetPort: metrics
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: public-js-ceramic-dev-service
+  labels:
+    App: JsCeramicDev
+spec:
+  type: LoadBalancer
+  selector:
+    App: JsCeramicDev
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: http-api
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -91,19 +121,3 @@ spec:
                 name: public-js-ceramic-dev-service
                 port:
                   number: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: public-js-ceramic-dev-service
-  labels:
-    App: JsCeramicDev
-spec:
-  type: LoadBalancer
-  selector:
-    App: JsCeramicDev
-  ports:
-    - protocol: TCP
-      name: http
-      port: 80
-      targetPort: 7007
diff --git a/ceramic-k8s/ceramic_deployment_prod.yaml b/ceramic-k8s/ceramic_deployment_prod.yaml
@@ -53,7 +53,7 @@ spec:
         vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/production/db
         vault.hashicorp.com/agent-inject-template-config: |
           {{- with secret "secrets/desci-server/production/db" -}}
-          echo "{\"anchor\":{\"auth-method\":\"did\",\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"node\":{\"privateSeedUrl\":\"inplace:ed25519#{{ .Data.ceramic_cas_pkey }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-prod-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"metrics-exporter-enabled\":false,\"metrics-port\":9090},\"network\":{\"name\":\"mainnet\"},\"state-store\":{\"mode\":\"fs\",\"local-directory\":\"/root/.ceramic/statestore\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"models\":[]}}" > daemon.config.json
+          echo "{\"anchor\":{\"auth-method\":\"did\",\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"node\":{\"privateSeedUrl\":\"inplace:ed25519#{{ .Data.ceramic_cas_pkey }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-prod-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"prometheus-exporter-enabled\":true,\"prometheus-exporter-port\":9464},\"network\":{\"name\":\"mainnet\"},\"state-store\":{\"mode\":\"fs\",\"local-directory\":\"/root/.ceramic/statestore\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true}}" > daemon.config.json
           export NODE_ENV=production
           export CERAMIC_PUBSUB_QPS_LIMIT=500
           echo "envset"; 
@@ -62,7 +62,7 @@ spec:
         App: JsCeramicProd
     spec:
       containers:
-        - image: ceramicnetwork/js-ceramic:5.16.0
+        - image: ceramicnetwork/js-ceramic:6.5.0
           name: js-ceramic-prod
           command: ["/bin/bash", "-c"]
           volumeMounts:
@@ -73,12 +73,14 @@ spec:
           ports:
             - containerPort: 7007
               name: http-api
+            - containerPort: 9464
+              name: metrics
           resources:
             limits:
-              cpu: "0.7"
+              cpu: 2
               memory: 4Gi
             requests:
-              cpu: "0.6"
+              cpu: 1
               memory: 4Gi
           # # restart pod after failureThreshold*periodSeconds total seconds
           livenessProbe:
@@ -107,9 +109,39 @@ spec:
         - name: ceramic-prod-persistent-storage
           persistentVolumeClaim:
             claimName: ceramic-prod-persistent-storage-pvc
-
       serviceAccountName: "vault-auth"
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: js-ceramic-prod-service-internal
+  labels:
+    App: JsCeramicProd
+spec:
+  type: ClusterIP
+  selector:
+    App: JsCeramicProd
+  ports:
+    - name: metrics
+      port: 9464
+      targetPort: metrics
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: public-js-ceramic-prod-service
+  labels:
+    App: JsCeramicProd
+spec:
+  type: LoadBalancer
+  selector:
+    App: JsCeramicProd
+  ports:
+    - protocol: TCP
+      name: http
+      port: 80
+      targetPort: http-api
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -128,19 +160,3 @@ spec:
                 name: public-js-ceramic-prod-service
                 port:
                   number: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: public-js-ceramic-prod-service
-  labels:
-    App: JsCeramicProd
-spec:
-  type: LoadBalancer
-  selector:
-    App: JsCeramicProd
-  ports:
-    - protocol: TCP
-      name: http
-      port: 80
-      targetPort: 7007