DDEX publisher module scaffolding (#7381)

.circleci/src/workflows/discovery.yml

@@ -98,6 +98,13 @@ jobs:
       filters:
         branches:
           only: main
+  - push-docker-image:
+      name: push-ddex-publisher
+      context: [Vercel, dockerhub]
+      service: ddex-publisher
+      filters:
+        branches:
+          only: main
 
   - lint-discovery-provider:
       name: lint-discovery-provider
@@ -233,6 +240,15 @@ jobs:
           only: main
       requires:
         - push-ddex-ingester
+  - push-arm-image:
+      name: push-ddex-publisher-arm
+      context: [Vercel, dockerhub]
+      service: ddex-publisher
+      filters:
+        branches:
+          only: main
+      requires:
+        - push-ddex-publisher
 
   # Deploy audius-protocol `main` branch (stage)
   - deploy-stage-nodes:
@@ -254,6 +270,7 @@ jobs:
         - push-healthz
         - push-ddex-web
         - push-ddex-ingester
+        - push-ddex-publisher
         # uncomment when arm builds are stable
         # - push-discovery-provider-arm
         # - push-discovery-provider-notifications-arm
@@ -267,6 +284,7 @@ jobs:
         # - push-uptime-arm
         # - push-ddex-web-arm
         # - push-ddex-ingester-arm
+        # - push-ddex-publisher-arm
 
       filters:
         branches:

.circleci/src/workflows/release.yml

@@ -80,6 +80,11 @@ jobs:
       context: [GCP, dockerhub, slack-secrets]
       service: ddex-ingester
       notify_slack_on_failure: true
+  - push-docker-image:
+      name: push-ddex-publisher
+      context: [GCP, dockerhub, slack-secrets]
+      service: ddex-publisher
+      notify_slack_on_failure: true
 
   - push-arm-image:
       name: push-discovery-provider-arm
@@ -166,6 +171,12 @@ jobs:
       service: ddex-ingester
       requires:
         - push-ddex-ingester
+  - push-arm-image:
+      name: push-ddex-publisher-arm
+      context: [Vercel, dockerhub]
+      service: ddex-publisher
+      requires:
+        - push-ddex-publisher
 
   - release-github-draft-trigger:
       type: approval
@@ -208,6 +219,7 @@ jobs:
         - push-uptime
         - push-ddex-web
         - push-ddex-ingester
+        - push-ddex-publisher
         # uncomment these when arm builds are stable
         # - push-identity-service-arm
         # - push-mediorum-arm
@@ -223,6 +235,7 @@ jobs:
         # - push-uptime-arm
         # - push-ddex-web-arm
         # - push-ddex-ingester-arm
+        # - push-ddex-publisher-arm
 
   - deploy-foundation-nodes-trigger:
       requires:

dev-tools/compose/docker-compose.ddex.yml

@@ -10,6 +10,8 @@ services:
         app_name: ddex-web
         TURBO_TEAM: '${TURBO_TEAM}'
         TURBO_TOKEN: '${TURBO_TOKEN}'
+    ports:
+      - "9000:9000"
     networks:
       - ddex-network
     profiles:
@@ -63,6 +65,27 @@ services:
     profiles:
       - ddex
 
+  ddex-publisher:
+    container_name: ddex-publisher
+    build:
+      context: ${PROJECT_ROOT}
+      dockerfile: ${PROJECT_ROOT}/packages/ddex/publisher/Dockerfile
+      args:
+        app_name: ddex-publisher
+        TURBO_TEAM: '${TURBO_TEAM}'
+        TURBO_TOKEN: '${TURBO_TOKEN}'
+    environment:
+      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    depends_on:
+      ddex-mongo:
+        condition: service_healthy
+    ports:
+      - "9001:9001"
+    networks:
+      - ddex-network
+    profiles:
+      - ddex
+
   ddex-mongo:
     container_name: ddex-mongo
     image: mongo:latest

package.json

@@ -8,6 +8,7 @@
     "protocol-dashboard",
     "packages/ddex/webapp/server",
     "packages/ddex/webapp/client",
+    "packages/ddex/publisher",
     "packages/es-indexer"
   ],
   "private": true,

packages/ddex/publisher/.env.stage

@@ -0,0 +1,3 @@
+NODE_ENV='stage'
+DDEX_KEY='49d5e13d355709b615b7cce7369174fb240b6b39'
+DDEX_SECRET='2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152'

packages/ddex/publisher/.eslintrc.cjs

@@ -0,0 +1,30 @@
+module.exports = {
+  root: true,
+  env: { node: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'prettier',
+    'plugin:prettier/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['prettier'],
+  rules: {
+    '@typescript-eslint/no-unsafe-call': 'off',
+    '@typescript-eslint/no-unsafe-assignment': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+    '@typescript-eslint/no-non-null-assertion': 'off',
+    '@typescript-eslint/no-empty-function': 'off',
+    'prettier/prettier': ['error', {
+      singleQuote: true,
+      semi: false,
+      useTabs: false,
+      tabWidth: 2,
+      trailingComma: 'es5',
+      printWidth: 80,
+      bracketSpacing: true,
+      arrowParens: 'always',
+    }],
+  },
+};

packages/ddex/publisher/Dockerfile

@@ -0,0 +1,60 @@
+FROM node:18-alpine AS base
+
+ARG TURBO_TEAM
+ENV TURBO_TEAM=$TURBO_TEAM
+
+ARG TURBO_TOKEN
+ENV TURBO_TOKEN=$TURBO_TOKEN
+
+# First stage: Set up a minimal monorepo
+FROM base AS turbo-builder
+
+RUN apk add --no-cache libc6-compat
+RUN apk update
+
+WORKDIR /app
+RUN npm install turbo --global
+
+COPY . .
+RUN turbo prune --scope=@audius/ddex-publisher --scope=@audius/ddex-publisher --docker
+
+# Second stage: Install and build client and server dists
+FROM base AS app-builder
+
+WORKDIR /app
+RUN apk add --no-cache python3 py3-pip make g++ curl bash libc6-compat git
+RUN apk update
+
+# First install dependencies (as they change less often)
+COPY .gitignore .gitignore
+COPY --from=turbo-builder /app/out/json/ .
+COPY --from=turbo-builder /app/out/package-lock.json ./package-lock.json
+COPY --from=turbo-builder /app/scripts ./scripts
+
+RUN CI=true npm i
+
+# Build the app and its dependencies
+COPY --from=turbo-builder /app/out/full/ .
+COPY svgr-template.js svgr-template.js
+COPY turbo.json turbo.json
+RUN npx turbo run build --filter=@audius/ddex-publisher --filter=@audius/ddex-publisher
+
+# Make a smaller image by removing all src directories (except for in node_modules)
+RUN find packages -path '*/node_modules/*' -prune -o -name 'src' -type d -exec rm -rf {} +
+
+# Final stage: Create a runnable image
+FROM node:18-alpine AS runner
+
+RUN apk add --no-cache python3 py3-pip make g++ curl bash libc6-compat
+RUN apk update
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nodejs
+USER nodejs
+
+WORKDIR /app
+COPY --from=app-builder --chown=nodejs:nodejs /app .
+WORKDIR /app/packages/ddex/publisher
+
+EXPOSE 9001
+CMD ["node", "dist/index.js"]

packages/ddex/publisher/README.md

@@ -0,0 +1,9 @@
+# Audius DDEX Publisher
+
+Server that publishes DDEX entities queued for release by the DDEX ingester.
+
+### Local Dev
+Run the server:
+1. Make sure you can connect to mongo at `mongodb://mongo:mongo@localhost:27017/ddex` by doing: `docker run --name ddex-mongo -p 27017:27017 -e MONGO_INITDB_ROOT_USERNAME=mongo -e MONGO_INITDB_ROOT_PASSWORD=mongo -d mongo`
+2. At the monorepo root: `npm i`
+3. At packages/ddex/publisher: `npm run dev:[stage|prod]`

packages/ddex/publisher/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@audius/ddex-publisher",
+  "version": "0.0.1",
+  "description": "Server that publishes DDEX entities queued for release",
+  "main": "index.js",
+  "scripts": {
+    "build": "tsc",
+    "start:prod": "NODE_ENV=production node dist/index.js",
+    "start": "nodemon src/index.ts",
+    "dev:stage": "NODE_ENV=stage turbo run start --filter=@audius/ddex-publisher",
+    "dev:prod": "NODE_ENV=production turbo run start --filter=@audius/ddex-publisher",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "lint:fix": "eslint --fix src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "verify": "concurrently \"npm:build\" \"npm:lint\""
+  },
+  "keywords": [],
+  "author": "Audius",
+  "dependencies": {
+    "mongodb": "6.3.0",
+    "mongoose": "8.1.0",
+    "dotenv": "16.3.1"
+  },
+  "devDependencies": {
+    "@types/node": "20.11.10",
+    "typescript": "5.3.3"
+  }
+}

packages/ddex/publisher/src/app.ts

@@ -0,0 +1,18 @@
+import express, { Express, Request, Response } from 'express'
+
+export default function createApp() {
+  /*
+   * Setup app
+   */
+  const app: Express = express()
+
+  /*
+   * Define API routes
+   */
+
+  app.get('/api/health_check', (_req: Request, res: Response) => {
+    res.status(200).send('DDEX publisher is alive!')
+  })
+
+  return app
+}

packages/ddex/publisher/src/index.ts

@@ -0,0 +1,29 @@
+import dotenv from 'dotenv'
+import path from 'path'
+
+// Load env vars based on NODE_ENV
+const envFile = process.env.NODE_ENV === 'stage' ? '.env.stage' : '.env'
+dotenv.config({ path: path.resolve(process.cwd(), envFile) })
+
+import createApp from './app'
+import { dialDb } from './services/dbService'
+
+const port = process.env.DDEX_PORT || 9001
+
+;(async () => {
+  try {
+    const dbUrl =
+      process.env.DDEX_MONGODB_URL ||
+      'mongodb://mongo:mongo@localhost:27017/ddex?authSource=admin'
+    await dialDb(dbUrl)
+
+    const app = createApp()
+
+    app.listen(port, () => {
+      console.log(`[server]: Server is running at http://localhost:${port}`)
+    })
+  } catch (error) {
+    console.error('Failed to initialize:', error)
+    process.exit(1)
+  }
+})()

packages/ddex/publisher/src/services/dbService.ts

@@ -0,0 +1,6 @@
+import mongoose from 'mongoose'
+
+export const dialDb = async (dbUrl: string) => {
+  await mongoose.connect(dbUrl)
+  console.log('MongoDB connected...')
+}