ddex yaml time (#8515)

AudiusProject · May 20, 2024 · 981954f · 981954f
1 parent 665938a
commit 981954f
Show file tree

Hide file tree

Showing 14 changed files with 100 additions and 85 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -475,6 +475,7 @@ workflows:
             packages/ddex/webapp/.* run-ddex-webapp-workflow true
             packages/ddex/webapp/.* run-ddex-e2e-workflow true
             packages/ddex/ingester/.* run-ddex-e2e-workflow true
+            packages/ddex/processor/.* run-ddex-processor-workflow true
             packages/ddex/publisher/.* run-ddex-publisher-workflow true
             eth-contracts/.* run-eth-contracts-workflow true
             monitoring/healthz/.* run-healthz-workflow true
@@ -487,6 +488,7 @@ workflows:
             .circleci/.* run-protocol-dashboard-workflow true
             .circleci/.* run-ddex-webapp-workflow true
             .circleci/.* run-ddex-e2e-workflow true
+            .circleci/.* run-ddex-processor-workflow true
             .circleci/.* run-ddex-publisher-workflow true
             .circleci/.* run-healthz-workflow true
             .circleci/.* run-contracts-workflow true
@@ -515,6 +517,7 @@ workflows:
             packages/libs/.* run-identity-workflow true
             packages/libs/.* run-protocol-dashboard-workflow true
             packages/libs/.* run-ddex-webapp-workflow true
+            packages/libs/.* run-ddex-processor-workflow true
             packages/libs/.* run-ddex-publisher-workflow true
             packages/libs/.* run-sdk-workflow true
             packages/libs/.* run-web-workflow true
@@ -564,6 +567,7 @@ workflows:
             packages/ddex/webapp/.* run-ddex-webapp-workflow true
             packages/ddex/webapp/.* run-ddex-e2e-workflow true
             packages/ddex/ingester/.* run-ddex-e2e-workflow true
+            packages/ddex/processor/.* run-ddex-processor-workflow true
             packages/ddex/publisher/.* run-ddex-publisher-workflow true
             packages/create-audius-app/.* run-create-audius-app-workflow true
             protocol-dashboard/.* run-protocol-dashboard-workflow true

diff --git a/.circleci/src/@continue_config.yml b/.circleci/src/@continue_config.yml
@@ -70,6 +70,9 @@ parameters:
   run-ddex-webapp-workflow:
     type: boolean
     default: false
+  run-ddex-processor-workflow:
+    type: boolean
+    default: false
   run-ddex-publisher-workflow:
     type: boolean
     default: false

diff --git a/.circleci/src/commands/@docker-commands.yml b/.circleci/src/commands/@docker-commands.yml
@@ -19,7 +19,14 @@ docker-tag-images:
       enum: ['prerelease', 'edge', 'current', '__version__']
     service:
       type: enum
-      enum: ['discovery-provider', 'creator-node', 'identity-service', 'ddex', 'all']
+      enum:
+        [
+          'discovery-provider',
+          'creator-node',
+          'identity-service',
+          'ddex',
+          'all'
+        ]
   steps:
     - run:
         name: Get crane
@@ -58,6 +65,7 @@ docker-tag-images:
           ddex=(
             ddex-ingester
             ddex-publisher
+            ddex-processor
             ddex-web
           )
 

diff --git a/.circleci/src/jobs/@ddex-jobs.yml b/.circleci/src/jobs/@ddex-jobs.yml
@@ -55,3 +55,32 @@ ddex-publisher-init:
           - node_modules
           - packages/ddex/node_modules
           - packages/ddex/publisher/node_modules
+
+ddex-processor-init:
+  working_directory: ~/audius-protocol
+  docker:
+    - image: cimg/node:18.17
+  steps:
+    - checkout
+    - attach_workspace:
+        at: ./
+
+    # Download and cache dependencies
+    - create_concatenated_patch_file:
+        filename: combined-patch-file.txt
+
+    - restore_cache:
+        keys:
+          - cache-{{ checksum "package-lock.json" }}-{{ checksum "combined-patch-file.txt" }}
+
+    - run:
+        name: lint & typecheck
+        command: |
+          npx turbo run verify --filter=@audius/ddex-processor
+
+    - persist_to_workspace:
+        root: ./
+        paths:
+          - node_modules
+          - packages/ddex/node_modules
+          - packages/ddex/processor/node_modules
diff --git a/.circleci/src/workflows/ddex-processor.yml b/.circleci/src/workflows/ddex-processor.yml
@@ -0,0 +1,3 @@
+when: << pipeline.parameters.run-ddex-processor-workflow >>
+jobs:
+  - ddex-processor-init
diff --git a/.circleci/src/workflows/ddex-stage.yml b/.circleci/src/workflows/ddex-stage.yml
@@ -26,6 +26,13 @@ jobs:
       filters:
         branches:
           only: main
+  - push-docker-image:
+      name: push-ddex-processor
+      context: [Vercel, dockerhub]
+      service: ddex-processor
+      filters:
+        branches:
+          only: main
   - push-docker-image:
       name: push-ddex-publisher
       context: [Vercel, dockerhub]

diff --git a/.circleci/src/workflows/discovery.yml b/.circleci/src/workflows/discovery.yml
@@ -112,6 +112,13 @@ jobs:
       filters:
         branches:
           only: main
+  - push-docker-image:
+      name: push-ddex-processor
+      context: [Vercel, dockerhub]
+      service: ddex-processor
+      filters:
+        branches:
+          only: main
   - push-docker-image:
       name: push-ddex-publisher
       context: [Vercel, dockerhub]
@@ -306,6 +313,7 @@ jobs:
         - push-healthz
         - push-ddex-web
         - push-ddex-ingester
+        - push-ddex-processor
         - push-ddex-publisher
         # uncomment when arm builds are stable
         # - push-discovery-provider-arm

diff --git a/.circleci/src/workflows/release.yml b/.circleci/src/workflows/release.yml
@@ -90,6 +90,11 @@ jobs:
       context: [GCP, dockerhub, slack-secrets]
       service: ddex-ingester
       notify_slack_on_failure: true
+  - push-docker-image:
+      name: push-ddex-processor
+      context: [GCP, dockerhub, slack-secrets]
+      service: ddex-processor
+      notify_slack_on_failure: true
   - push-docker-image:
       name: push-ddex-publisher
       context: [GCP, dockerhub, slack-secrets]
@@ -227,6 +232,7 @@ jobs:
         - push-uptime
         - push-ddex-web
         - push-ddex-ingester
+        - push-ddex-processor
         - push-ddex-publisher
         # uncomment these when arm builds are stable
         # - push-identity-service-arm

diff --git a/dev-tools/compose/docker-compose.ddex.yml b/dev-tools/compose/docker-compose.ddex.yml
@@ -1,6 +1,23 @@
 version: '3.9'
 
 services:
+  ddex-processor:
+    container_name: ddex-processor
+    build:
+      context: ${PROJECT_ROOT}
+      dockerfile: ${PROJECT_ROOT}/packages/ddex/processor/Dockerfile
+      args:
+        app_name: ddex-processor
+        TURBO_TEAM: '${TURBO_TEAM}'
+        TURBO_TOKEN: '${TURBO_TOKEN}'
+    env_file: .env
+    ports:
+      - "8889:8989"
+    networks:
+      - ddex-network
+    profiles:
+      - ddex
+
   ddex-web:
     container_name: ddex-web
     build:

diff --git a/dev-tools/compose/docker-compose.yml b/dev-tools/compose/docker-compose.yml
@@ -225,6 +225,15 @@ services:
     profiles:
       - ddex
 
+  ddex-processor:
+    extends:
+      file: docker-compose.ddex.yml
+      service: ddex-processor
+    container_name: ddex-processor
+    <<: *common
+    profiles:
+      - ddex
+
   ddex-publisher:
     extends:
       file: docker-compose.ddex.yml

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/ddex/processor/Dockerfile b/packages/ddex/processor/Dockerfile
@@ -54,7 +54,7 @@ USER nodejs
 
 WORKDIR /app
 COPY --from=app-builder --chown=nodejs:nodejs /app .
-WORKDIR /app/packages/ddex/publisher
+WORKDIR /app/packages/ddex/processor
 
 EXPOSE 9001
 CMD ["npm", "run", "start:prod"]
diff --git a/packages/ddex/processor/package.json b/packages/ddex/processor/package.json
@@ -10,7 +10,7 @@
     "start": "tsx watch cli.ts server",
     "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "lint:fix": "eslint --fix src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
-    "verify": "concurrently \"npm:build\" \"npm:lint\"",
+    "verify": "tsc --noEmit",
     "test": "SQLITE_URL=:memory: vitest run",
     "test:watch": "SQLITE_URL=:memory: vitest"
   },
@@ -27,7 +27,7 @@
     "commander": "12.0.0",
     "decompress": "4.2.1",
     "dotenv": "16.3.1",
-    "file-type": "19.0.0",
+    "file-type": "16.5.3",
     "hono": "4.2.5",
     "web3": "4.3.0"
   },

diff --git a/packages/ddex/processor/src/server.ts b/packages/ddex/processor/src/server.ts
@@ -1,7 +1,7 @@
 import 'dotenv/config'
 
 import { serve } from '@hono/node-server'
-import { fileTypeFromBuffer } from 'file-type'
+import { fromBuffer as fileTypeFromBuffer } from 'file-type'
 import { Context, Hono } from 'hono'
 import { deleteCookie, getSignedCookie, setSignedCookie } from 'hono/cookie'
 import { html } from 'hono/html'