Implements new server/agent architecture, exposes realtime log file i…

…nfo (#113) * Implements new server/agent architecture with small agents designed to run on each node * Agents expose gRPC server with LogMetadataService that sends log file info to client * Implements CI workflow for new architecture Signed-off-by: Andres Morey <andresmarcel@gmail.com>
kubetail-org · Sep 24, 2024 · 66fde8c · 66fde8c
1 parent 4e97f57
commit 66fde8c
Show file tree

Hide file tree

Showing 105 changed files with 10,136 additions and 2,750 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,9 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.6'
+          go-version: '1.22.7'
+          cache: false
       - name: Run linter
         working-directory: ./backend
         run: |
@@ -22,25 +23,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.6'
+          go-version: '1.22.7'
+          cache: false
       - name: Run tests
         working-directory: ./backend
         run: |
-          go test -race ./...
+          go test -race github.com/kubetail-org/kubetail/backend/...
 
   backend-vet:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.6'
+          go-version: '1.22.7'
+          cache: false
       - name: Run code vetter
         working-directory: ./backend
         run: |
-          go vet ./...
+          go vet github.com/kubetail-org/kubetail/backend/...
 
   frontend-lint:
     runs-on: ubuntu-latest
@@ -51,7 +54,7 @@ jobs:
           node-version: 20
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 9
       - name: Set up and run linter
         working-directory: ./frontend
         run: |
@@ -67,7 +70,7 @@ jobs:
           node-version: 20
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 9
       - name: Set up and run tests
         working-directory: ./frontend
         run: |
@@ -83,14 +86,14 @@ jobs:
           node-version: 20
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 9
       - name: Set up and run build script
         working-directory: ./frontend
         run: |
           pnpm install
           pnpm build
 
-  build-amd64:
+  build-amd64-server:
     needs:
       - backend-lint
       - backend-test
@@ -106,10 +109,51 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: build/package/Dockerfile.server
           push: false
-          tags: kubetail:latest
+          tags: kubetail-server:latest
 
-  build-arm64:
+  build-amd64-agent:
+    needs:
+      - backend-lint
+      - backend-test
+      - backend-vet
+      - frontend-lint
+      - frontend-test
+      - frontend-build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/package/Dockerfile.agent
+          push: false
+          tags: kubetail-agent:latest
+
+  build-arm64-sever:
+    needs:
+      - backend-lint
+      - backend-test
+      - backend-vet
+      - frontend-lint
+      - frontend-test
+      - frontend-build
+    runs-on: arm64-ubuntu-22
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/package/Dockerfile.server
+          push: false
+          tags: kubetail-server:latest
+
+  build-arm64-agent:
     needs:
       - backend-lint
       - backend-test
@@ -125,5 +169,6 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: build/package/Dockerfile.agent
           push: false
-          tags: kubetail:latest
+          tags: kubetail-agent:latest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -6,7 +6,7 @@ on:
       - '**'
 
 jobs:
-  build-and-publish-amd64:
+  build-and-publish-amd64-server:
     runs-on: ubuntu-latest
     steps:
       - name: Get tag name
@@ -25,10 +25,57 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: build/package/Dockerfile.server
           push: true
-          tags: kubetail/kubetail:${{ steps.tagName.outputs.tag }}-amd64
+          tags: kubetail/kubetail-server:${{ steps.tagName.outputs.tag }}-amd64
 
-  build-and-publish-arm64:
+  build-and-publish-amd64-agent:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get tag name
+        uses: olegtarasov/get-tag@2.1.3
+        id: tagName
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3        
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/package/Dockerfile.agent
+          push: true
+          tags: kubetail/kubetail-agent:${{ steps.tagName.outputs.tag }}-amd64
+
+  build-and-publish-arm64-server:
+    runs-on: arm64-ubuntu-22
+    steps:
+      - name: Get tag name
+        uses: olegtarasov/get-tag@2.1.3
+        id: tagName
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3        
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: build/package/Dockerfile.server
+          push: true
+          tags: kubetail/kubetail-server:${{ steps.tagName.outputs.tag }}-arm64
+
+  build-and-publish-arm64-agent:
     runs-on: arm64-ubuntu-22
     steps:
       - name: Get tag name
@@ -47,12 +94,53 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: build/package/Dockerfile.agent
           push: true
-          tags: kubetail/kubetail:${{ steps.tagName.outputs.tag }}-arm64
+          tags: kubetail/kubetail-agent:${{ steps.tagName.outputs.tag }}-arm64
+
+  create-and-publish-manifest-server:
+    runs-on: ubuntu-latest
+    needs: [build-and-publish-amd64-server, build-and-publish-arm64-server]
+    steps:
+      - name: Get tag name
+        uses: olegtarasov/get-tag@2.1.3
+        id: tagName
+      - name: 'Setup jq'
+        uses: dcarbone/install-jq-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3        
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Create and push manifest
+        run: |
+          docker buildx imagetools create -t kubetail/kubetail-server:${{ steps.tagName.outputs.tag }} \
+            kubetail/kubetail-server:${{ steps.tagName.outputs.tag }}-amd64 \
+            kubetail/kubetail-server:${{ steps.tagName.outputs.tag }}-arm64
+      - name: Fetch docker token
+        run: |
+          TOKEN=$(curl -X POST "https://hub.docker.com/v2/users/login" -H "Content-Type: application/json" -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' | jq -r '.token')
+          echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+      - name: Delete extra arch manifests
+        run: |
+          declare -a archs=("amd64" "arm64")
+          for arch in "${archs[@]}"
+          do
+            RESPONSE=$(curl -s -w "%{http_code}" \
+              -X DELETE \
+              -H "Authorization: Bearer $TOKEN" \
+              "https://hub.docker.com/v2/repositories/kubetail/kubetail-server/tags/${{ steps.tagName.outputs.tag }}-$arch")
+            if [ "$RESPONSE" -ne 204 ]; then
+              echo "DELETE for $arch failed with status $RESPONSE"
+              exit 1
+            fi
+          done
 
-  create-and-publish-manifest:
+  create-and-publish-manifest-agent:
     runs-on: ubuntu-latest
-    needs: [build-and-publish-amd64, build-and-publish-arm64]
+    needs: [build-and-publish-amd64-agent, build-and-publish-arm64-agent]
     steps:
       - name: Get tag name
         uses: olegtarasov/get-tag@2.1.3
@@ -68,9 +156,9 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Create and push manifest
         run: |
-          docker buildx imagetools create -t kubetail/kubetail:${{ steps.tagName.outputs.tag }} \
-            kubetail/kubetail:${{ steps.tagName.outputs.tag }}-amd64 \
-            kubetail/kubetail:${{ steps.tagName.outputs.tag }}-arm64
+          docker buildx imagetools create -t kubetail/kubetail-agent:${{ steps.tagName.outputs.tag }} \
+            kubetail/kubetail-agent:${{ steps.tagName.outputs.tag }}-amd64 \
+            kubetail/kubetail-agent:${{ steps.tagName.outputs.tag }}-arm64
       - name: Fetch docker token
         run: |
           TOKEN=$(curl -X POST "https://hub.docker.com/v2/users/login" -H "Content-Type: application/json" -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' | jq -r '.token')
@@ -83,7 +171,7 @@ jobs:
             RESPONSE=$(curl -s -w "%{http_code}" \
               -X DELETE \
               -H "Authorization: Bearer $TOKEN" \
-              "https://hub.docker.com/v2/repositories/kubetail/kubetail/tags/${{ steps.tagName.outputs.tag }}-$arch")
+              "https://hub.docker.com/v2/repositories/kubetail/kubetail-agent/tags/${{ steps.tagName.outputs.tag }}-$arch")
             if [ "$RESPONSE" -ne 204 ]; then
               echo "DELETE for $arch failed with status $RESPONSE"
               exit 1

diff --git a/.gitignore b/.gitignore
@@ -23,7 +23,10 @@ lerna-debug.log*
 # build
 /frontend/dist
 /frontend/dist-ssr
-/backend/website
+/backend/server/website
+
+# tilt (custom)
+.tilt
 
 # tilt (custom)
 .tilt