Merge pull request #400 from desci-labs/faster-builds

faster builds
desci-labs · Jun 25, 2024 · 1bd70a2 · 1bd70a2
2 parents f1f7ff9 + ba674f8
commit 1bd70a2
Show file tree

Hide file tree

Showing 11 changed files with 143 additions and 19 deletions.
diff --git a/.env.test b/.env.test
@@ -2,7 +2,7 @@ NODE_ENV=test
 
 PORT=5421
 
-IPFS_NODE_URL=http://host.docker.internal:5002
+IPFS_NODE_URL=http://host.docker.internal:5003
 PUBLIC_IPFS_RESOLVER=https://ipfs.io
 
 ### Database - Postgres

diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml
@@ -71,9 +71,9 @@ jobs:
             exit 1
           fi
 
-  build-and-push:
-    needs: build-and-test
-    name: Build and deploy
+  build-and-push-images:
+    # we build and push for every commit, even if tests pass, that way when tests pass deployment is short (run test + build in parallel)
+    name: Build and push images
     runs-on: ubuntu-latest
     steps:
       - uses: hashicorp/setup-terraform@v1
@@ -125,7 +125,7 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE-dev:latest \
+            -t $CONTAINER_IMAGE-dev:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
              .
 
@@ -134,7 +134,7 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE-demo:latest \
+            -t $CONTAINER_IMAGE-demo:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo \
              .
 
@@ -143,7 +143,7 @@ jobs:
         run: |
           # Build and tag the image
           docker build \
-            -t $CONTAINER_IMAGE:latest \
+            -t $CONTAINER_IMAGE:${{ github.sha }} \
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \
              .
 
@@ -156,28 +156,85 @@ jobs:
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
           docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
-          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
 
       - name: Push (DEMO)
         if: github.ref == 'refs/heads/demo'
         run: |
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
           docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
-          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
 
       - name: Push (PROD)
         if: github.ref == 'refs/heads/main'
         run: |
           # Push image to AWS ECR
           aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
           docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
-          docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
+
+  deploy:
+    name: Deploy desci-server
+    needs:
+      - build-and-test
+      - build-and-push-images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      # Only push to registry on dev
+      - name: Push (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+
+      - name: Push (DEMO)
+        if: github.ref == 'refs/heads/demo'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+
+      - name: Push (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
+          docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
           docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
 
       - name: Deploy to EKS (DEV)

diff --git a/desci-repo/.env.test b/desci-repo/.env.test
@@ -4,7 +4,7 @@ NODE_ENV=test
 PORT=5484 # mapped to 5486
 WS_PORT=5445 # mapped to 5446
 
-IPFS_NODE_URL=http://host.docker.internal:5002
+IPFS_NODE_URL=http://host.docker.internal:5003
 PUBLIC_IPFS_RESOLVER=https://ipfs.io
 ### Database - Postgres
 PG_HOST=host.docker.internal

diff --git a/desci-repo/src/config.ts b/desci-repo/src/config.ts
@@ -2,5 +2,5 @@ export const PUBLIC_IPFS_PATH =
   process.env.NODE_ENV === 'dev'
     ? `http://host.docker.internal:8089/ipfs`
     : process.env.NODE_ENV === 'test'
-      ? 'http://host.docker.internal:8090/ipfs'
+      ? 'http://host.docker.internal:8091/ipfs'
       : 'https://ipfs.desci.com/ipfs';
diff --git a/desci-server/package.json b/desci-server/package.json
@@ -38,7 +38,8 @@
     "prettier": "prettier --config .prettierrc --write './**/*.{ts,js,css,scss,json,md}'",
     "test:destructive": "NODE_OPTIONS=\"--experimental-specifier-resolution=node --loader=ts-node/esm\" mocha --colors --require ts-node/register 'test/integration/**/*.test.ts' --timeout 20000 --exit",
     "test:destructive:debug": "yarn test:destructive --inspect=0.0.0.0:9227",
-    "test": "yarn docker:test; export EXIT=$(echo $?); docker-compose  --file ../docker-compose.test.yml --compatibility down; exit $EXIT",
+    "test:check": "npx tsc --project ./tsconfig-test.json",
+    "test": "yarn test:check && yarn docker:test; export EXIT=$(echo $?); docker-compose  --file ../docker-compose.test.yml --compatibility down; exit $EXIT",
     "coverage:destructive": "nyc --all --parser-plugins='[\"importAssertions\"]' -r lcov -e .ts -x \"*.test.ts\" npm run test:destructive",
     "coverage:destructive:debug": "nyc --all --parser-plugins='[\"importAssertions\"]' -r lcov -e .ts -x \"*.test.ts\" npm run test:destructive:debug",
     "commit": "git-cz",

diff --git a/desci-server/src/config/index.ts b/desci-server/src/config/index.ts
@@ -4,7 +4,7 @@ export const PUBLIC_IPFS_PATH =
   process.env.NODE_ENV === 'dev'
     ? `http://host.docker.internal:8089/ipfs`
     : process.env.NODE_ENV === 'test'
-      ? 'http://host.docker.internal:8090/ipfs'
+      ? 'http://host.docker.internal:8091/ipfs'
       : 'https://ipfs.desci.com/ipfs';
 
 export const MEDIA_SERVER_API_URL = process.env.NODES_MEDIA_SERVER_URL;

diff --git a/desci-server/src/server.ts b/desci-server/src/server.ts
@@ -122,6 +122,10 @@ class AppServer {
     this.app.use(cookieParser());
     this.app.set('trust proxy', 2); // detect AWS ELB IP + cloudflare
 
+    this.app.get('/test-update', (_req, res) => {
+      res.status(200).json({ status: 'ok', updated: true });
+    });
+
     this.#attachRouteHandlers();
 
     // catch 404 errors and forward to error handler

diff --git a/desci-server/test/integration/temp.test.ts b/desci-server/test/integration/temp.test.ts
@@ -0,0 +1,23 @@
+import 'mocha';
+import { User } from '@prisma/client';
+import { expect } from 'chai';
+import supertest from 'supertest';
+
+import { prisma } from '../../src/client.js';
+import { generateAccessToken } from '../../src/controllers/auth/magic.js';
+import { app } from '../../src/index.js';
+import { testingGenerateMagicCode } from '../util.js';
+
+describe('failing test', () => {
+  let request: supertest.SuperTest<supertest.Test>;
+
+  before(async () => {
+    request = supertest(app);
+  });
+
+  describe('should fail', () => {
+    it('should fail', async () => {
+      expect(1).to.equal(2);
+    });
+  });
+});
diff --git a/desci-server/tsconfig-test.json b/desci-server/tsconfig-test.json
@@ -0,0 +1,39 @@
+{
+  "compilerOptions": {
+    "module": "nodenext",
+    /**
+     * Implied by "module: nodenext":
+     * "moduleResolution": "nodenext",
+     * "esModuleInterop": true,
+     */
+    "target": "esnext",
+    // Recommended by docs, but a lot of errors
+    // "verbatimModuleSyntax": true,
+    "lib": [
+      "esnext",
+      "dom"
+    ],
+    "noEmit": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "resolveJsonModule": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "skipLibCheck": true,
+    // dagConcat.cjs requires this
+    "allowJs": true,
+    "inlineSourceMap": true,
+    "paths": {
+      // Overrides a built-in Response type
+      "express": [
+        "./src/types/express"
+      ]
+    },
+    "jsx": "react"
+  },
+  "include": [
+    "test/**/*.ts"
+  ],
+  "compileOnSave": true
+}
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -41,8 +41,8 @@ services:
     environment:
       IPFS_SWARM_KEY: "/key/swarm/psk/1.0.0/\n/base16/\n9d002c50635a479d29dcc0ccb49d862952a0dcc52baddd253167adcd496c8d04"
     ports:
-      - "5002:5001"
-      - "8090:8080"
+      - "5003:5001"
+      - "8091:8080"
     volumes:
       - ./local-data/test/ipfs:/data/ipfs
 

diff --git a/nodes-media/src/config/index.ts b/nodes-media/src/config/index.ts
@@ -2,5 +2,5 @@ export const PUBLIC_IPFS_PATH =
   process.env.NODE_ENV === 'dev'
     ? `http://host.docker.internal:8089/ipfs`
     : process.env.NODE_ENV === 'test'
-    ? 'http://host.docker.internal:8090/ipfs'
+    ? 'http://host.docker.internal:8091/ipfs'
     : 'https://ipfs.desci.com/ipfs';