Merge pull request #421 from desci-labs/develop

promote main
desci-labs · Jul 8, 2024 · b1af0b3 · b1af0b3
2 parents 4b73e66 + 4aaf98e
commit b1af0b3
Show file tree

Hide file tree

Showing 276 changed files with 28,072 additions and 8,628 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,11 +1,12 @@
+**/node_modules
+**/dist
+**/.vscode
+desci-server/log
 config
 dist
-./node_modules
 .git
 .env
 desci-art-viewer
 desci-dapp
-desci-server/node_modules
 local-data
 database
-desci-server/log
diff --git a/.env.example b/.env.example
@@ -69,6 +69,10 @@ AWS_SECRET_ACCESS_KEY=
 
 # for faucet
 HOT_WALLET_KEY=
+
+# for upgrading legacy dPID's (default is ganache account #1 for local dev)
+REGISTRY_OWNER_PKEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+
 # https://cso-classifier.internal
 CSO_CLASSIFIER_API=
 # vscode runnable
@@ -111,9 +115,6 @@ GOOGLE_DEV_API_KEY= # Unnecessary for now, not doing serverside 2step
 ## Configure RPC nodes (open an issue/ping us to access DeSci Labs' nodes)
 ETHEREUM_RPC_URL=http://host.docker.internal:8545
 
-# Use this for Goerli testnet
-# ETHEREUM_RPC_URL=https://eth-goerli.g.alchemy.com/v2/demo
-
 # Use this for Sepolia testnet
 # ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo
 
@@ -123,3 +124,22 @@ DPID_URL_OVERRIDE=https://dev-beta.dpid.org
 MUTE_PUBLISH_WORKER=false
 # SingleNodeLockServce
 MAX_LOCK_TIME=3600 # 1 hour
+
+DOI_PREFIX=10.62891
+CROSSREF_DOI_URL=https://doi.org
+
+# Cross ref api 
+CROSSREF_METADATA_API=https://test.crossref.org/servlet/deposit
+CROSSREF_ADMIN_API=https://test.crossref.org
+CROSSREF_EMAIL=
+CROSSREF_LOGIN=
+CROSSREF_PASSWORD=
+
+# Cross ref notification callback envs
+CROSSREF_NOTIFY_ENDPOINT=endpoint
+
+# automated metadata
+
+# Automated metadata
+AUTOMATED_METADATA_API=http://host.docker.internal:5005
+AUTOMATED_METADATA_API_KEY=
diff --git a/.env.test b/.env.test
@@ -2,7 +2,7 @@ NODE_ENV=test
 
 PORT=5421
 
-IPFS_NODE_URL=http://host.docker.internal:5002
+IPFS_NODE_URL=http://host.docker.internal:5003
 PUBLIC_IPFS_RESOLVER=https://ipfs.io
 
 ### Database - Postgres
@@ -72,4 +72,22 @@ VSCODE_ACCESS_TOKEN=
 NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454
 
 REPO_SERVICE_SECRET_KEY="m8sIy5BPygBcX3+ZmMVuAA10k6w59BSCZd+Z5+VLYm4="
-REPO_SERVER_URL=http://host.docker.internal:5485
+REPO_SERVER_URL=http://host.docker.internal:5485
+
+DOI_PREFIX=10.62891
+CROSSREF_DOI_URL=https://doi.org
+
+# Cross ref api 
+CROSSREF_METADATA_API=https://test.crossref.org/servlet/deposit
+CROSSREF_ADMIN_API=https://test.crossref.org
+CROSSREF_API_KEY=
+CROSSREF_EMAIL=
+CROSSREF_LOGIN=
+CROSSREF_PASSWORD=
+
+# Cross ref notification callback envs
+CROSSREF_NOTIFY_ENDPOINT=endpoint
+
+# Automated metadata
+AUTOMATED_METADATA_API=
+AUTOMATED_METADATA_API_KEY=
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -54,7 +54,7 @@ jobs:
           docker info
 
       - name: Install dependencies
-        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn && cd ../desci-repo && yarn
+        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
 
       - name: Stub contract
         run: |
@@ -63,7 +63,7 @@ jobs:
 
       - name: Run tests
         run: |
-          cd desci-server && export DOCKER_BUILDKIT=1 && yarn && yarn test
+          cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
           if [ $? -ne 0 ]; then
             exit 1
           fi
diff --git a/.github/workflows/build-isolated-media-server.yaml b/.github/workflows/build-isolated-media-server.yaml
@@ -55,7 +55,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.github/workflows/build-nodes-media.yaml b/.github/workflows/build-nodes-media.yaml
@@ -54,7 +54,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.github/workflows/build-repo-server.yaml b/.github/workflows/build-repo-server.yaml
@@ -90,7 +90,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.github/workflows/build-reverse-proxy.yaml b/.github/workflows/build-reverse-proxy.yaml
@@ -49,7 +49,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml
@@ -56,7 +56,7 @@ jobs:
           docker info
 
       - name: Install dependencies
-        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn
+        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines
 
       - name: Stub contract
         run: |
@@ -65,15 +65,15 @@ jobs:
 
       - name: Run tests
         run: |
-          cd desci-server && export DOCKER_BUILDKIT=1 && yarn && yarn test
+          cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
           echo "exit code $?"
           if [ $? -ne 0 ]; then
             exit 1
           fi
 
-  build-and-push:
-    needs: build-and-test
-    name: Build and deploy
+  build-and-push-images:
+    # we build and push for every commit, even if tests pass, that way when tests pass deployment is short (run test + build in parallel)
+    name: Build and push images
     runs-on: ubuntu-latest
     steps:
       - uses: hashicorp/setup-terraform@v1
@@ -102,7 +102,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       # - name: Check Docker Version
       #   run: docker --version
@@ -125,7 +124,7 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE-dev:latest \
+            -t $CONTAINER_IMAGE-dev:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
              .
 
@@ -134,7 +133,7 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE-demo:latest \
+            -t $CONTAINER_IMAGE-demo:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo \
              .
 
@@ -143,61 +142,98 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE:latest \
+            -t $CONTAINER_IMAGE:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \
              .
 
-      # Add additional steps here like scanning of image
-
-      # Only push to registry on dev
       - name: Push (DEV)
         if: github.ref == 'refs/heads/develop'
         run: |
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
-          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker tag $CONTAINER_IMAGE-dev:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
-          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
 
       - name: Push (DEMO)
         if: github.ref == 'refs/heads/demo'
         run: |
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
-          docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+          docker tag $CONTAINER_IMAGE-demo:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
-          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
 
       - name: Push (PROD)
         if: github.ref == 'refs/heads/main'
         run: |
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
-          docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
+          docker tag $CONTAINER_IMAGE:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
-          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
+
+  deploy:
+    name: Deploy desci-server
+    needs:
+      - build-and-test
+      - build-and-push-images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
 
       - name: Deploy to EKS (DEV)
         # uses: steebchen/kubectl@v2.0.0
         if: github.ref == 'refs/heads/develop'
         run: | # defaults to latest kubectl binary version
           kubectl apply -f desci-server/kubernetes/deployment_dev.yaml
           kubectl set image deployment/desci-server-dev desci-server-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
 
       - name: Deploy to EKS (DEMO)
         # uses: steebchen/kubectl@v2.0.0
         if: github.ref == 'refs/heads/demo'
         run: | # defaults to latest kubectl binary version
           kubectl set image deployment/desci-server-demo desci-server-demo=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }} --record
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
 
       - name: Deploy to EKS (PROD)
         if: github.ref == 'refs/heads/main'
         run: | # defaults to latest kubectl binary version
           kubectl apply -f desci-server/kubernetes/deployment_prod.yaml
           kubectl set image deployment/desci-server desci-server=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} --record
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
 
       - name: Verify EKS Deployment (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.github/workflows/deploy-staging-services.yaml b/.github/workflows/deploy-staging-services.yaml
@@ -46,7 +46,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag desci-media-isolated image (STAGING)
         run: |
@@ -103,7 +102,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag nodes-media-server image (STAGING)
         run: |
@@ -160,7 +158,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Stub contract
         run: |
@@ -222,7 +219,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Stub contract
         run: |
@@ -289,7 +285,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Build and tag the image (STAGING)
         run: |
@@ -307,7 +302,7 @@ jobs:
           docker tag $CONTAINER_IMAGE_PROXY-staging:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE_PROXY-staging:latest
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE_PROXY-staging:${{ github.sha }}
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE_PROXY-staging:latest
-      
+
       - name: Deploy to EKS (STAGING)
         run: | # defaults to latest kubectl binary version
           kubectl apply -f $CONTAINER_IMAGE_PROXY/deployment.yaml

diff --git a/.github/workflows/run-migrations-dev.yaml b/.github/workflows/run-migrations-dev.yaml
@@ -39,7 +39,6 @@ jobs:
           sudo apt-get install less
           echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
           aws sts get-caller-identity
-          kubectl describe deployments
 
       - name: Migrate DB (DEV)
         if: github.ref == 'refs/heads/develop'

diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-18.20.0
+20.8.1
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -9,7 +9,7 @@
       "type": "node",
       "request": "attach",
       "restart": true,
-      "port": 9229,
+      "port": 9228,
       "address": "localhost",
       "localRoot": "${workspaceFolder}",
       "remoteRoot": "/app"

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,7 +1,7 @@
 {
   "explorer.compactFolders": false,
   "files.exclude": {
-    "**/node_modules/**": true
+    "**/node_modules/**": false
   },
   "editor.formatOnSave": true, // Tell VSCode to format files on save
   "editor.defaultFormatter": "esbenp.prettier-vscode",